diff --git a/api/app/__init__.py b/api/app/__init__.py index fd4538b4..6a74460a 100644 --- a/api/app/__init__.py +++ b/api/app/__init__.py @@ -1,17 +1,21 @@ -from flask import Flask, g import logging.config -import sys import os -sys.path.append(os.path.join(os.path.dirname(__file__), "..")) -from flask_cors import CORS -from flask_sqlalchemy import SQLAlchemy -import pika +import sys import uuid -from celery import Celery + +import pika from app.decorators.restplus import api as api_rest_plus +from celery import Celery +from flask import Flask, g +from flask_cors import CORS from flask_login import LoginManager from flask_mail import Mail +from flask_sqlalchemy import SQLAlchemy + +from . import constants + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) """__________________________________producer for COMPUTE_______________________________________________________""" @@ -67,7 +71,6 @@ def is_calculation_module_alive(self,cm_id): -from . import constants dbGIS = SQLAlchemy() diff --git a/api/app/api_v1/__init__.py b/api/app/api_v1/__init__.py index 06835627..307a630d 100644 --- a/api/app/api_v1/__init__.py +++ b/api/app/api_v1/__init__.py @@ -1,12 +1,15 @@ from flask import Blueprint -api = Blueprint('api', __name__, url_prefix='/api') -from .stats import nsStats -from .heat_load_profile import load_profile_namespace + from ..decorators import etag +from . import computation_module from .computation_module import nsCM -from .users import nsUsers -from .upload import nsUpload +from .heat_load_profile import load_profile_namespace from .snapshot import nsSnapshot +from .stats import nsStats +from .upload import nsUpload +from .users import nsUsers + +api = Blueprint('api', __name__, url_prefix='/api') @api.before_request def before_request(): @@ -21,5 +24,4 @@ def after_request(rv): return rv -from . import computation_module # from . import computation, errors diff --git a/api/app/api_v1/computation_module.py b/api/app/api_v1/computation_module.py index 2f0e47ce..113e1bd8 100644 --- a/api/app/api_v1/computation_module.py +++ b/api/app/api_v1/computation_module.py @@ -1,37 +1,33 @@ -from celery.task.control import revoke - +import json +import os import signal -from flask import request, current_app,jsonify,redirect, \ - url_for,Response + +import requests +from app import CalculationModuleRpcClient, celery, helper, model +from app.constants import DATASET_DIRECTORY, UPLOAD_DIRECTORY +from app.decorators.exceptions import ComputationalModuleError, ValidationError from app.decorators.restplus import api -from app.decorators.serializers import compution_module_class, \ - input_computation_module, test_communication_cm, \ - compution_module_list, uploadfile, cm_id_input +from app.decorators.serializers import (cm_id_input, compution_module_class, + compution_module_list, + input_computation_module, + test_communication_cm, uploadfile) +from app.model import getCMList, getUI, register_calulation_module +from celery.task.control import revoke +from flask import (Response, current_app, jsonify, redirect, request, + send_file, send_from_directory, url_for) +from flask_restplus import Resource -from app.model import register_calulation_module,getUI,getCMList +from ..constants import DEFAULT_TIMEOUT +from ..decorators.timeout import timeout_signal_handler from ..helper import commands_in_array, run_command - from ..models.user import User -from app import model -from app import helper nsCM = api.namespace('cm', description='Operations related to statistisdscs') ns = nsCM -from flask_restplus import Resource -from app import celery -import requests -from app.decorators.exceptions import ValidationError, ComputationalModuleError -import os -import json -from flask import send_from_directory, send_file -from app.constants import UPLOAD_DIRECTORY, DATASET_DIRECTORY -from app import CalculationModuleRpcClient -from ..decorators.timeout import timeout_signal_handler -from ..constants import DEFAULT_TIMEOUT #TODO Add url to find right computation module diff --git a/api/app/api_v1/errors.py b/api/app/api_v1/errors.py index 59e5b4e4..a7463919 100644 --- a/api/app/api_v1/errors.py +++ b/api/app/api_v1/errors.py @@ -1,5 +1,6 @@ +from app.decorators.exceptions import ComputationalModuleError, ValidationError from flask import jsonify -from app.decorators.exceptions import ValidationError, ComputationalModuleError + from . import api diff --git a/api/app/api_v1/heat_load_profile.py b/api/app/api_v1/heat_load_profile.py index eea31d0c..3e44ecb4 100644 --- a/api/app/api_v1/heat_load_profile.py +++ b/api/app/api_v1/heat_load_profile.py @@ -1,21 +1,21 @@ -from app import celery import logging -from flask_restplus import Resource -from app.decorators.serializers import load_profile_aggregation_day_input, \ - load_profile_aggregation_curve_output, load_profile_aggregation_curve, load_profile_aggregation_hectares, \ - load_profile_aggregation_curve_hectares -from app.decorators.restplus import api -from app.decorators.exceptions import IntersectionException, HugeRequestException, ParameterException, RequestException -from app.models.heatloadQueries import HeatLoadProfile -from .. import helper -from ..decorators.timeout import return_on_timeout_endpoint - - import shapely.geometry as shapely_geom - +from app import celery +from app.decorators.exceptions import (HugeRequestException, + IntersectionException, + ParameterException, RequestException) +from app.decorators.restplus import api +from app.decorators.serializers import ( + load_profile_aggregation_curve, load_profile_aggregation_curve_hectares, + load_profile_aggregation_curve_output, load_profile_aggregation_day_input, + load_profile_aggregation_hectares) from app.models import generalData +from app.models.heatloadQueries import HeatLoadProfile +from flask_restplus import Resource +from .. import helper +from ..decorators.timeout import return_on_timeout_endpoint log = logging.getLogger(__name__) diff --git a/api/app/api_v1/snapshot.py b/api/app/api_v1/snapshot.py index b31717f4..49f6a1b1 100644 --- a/api/app/api_v1/snapshot.py +++ b/api/app/api_v1/snapshot.py @@ -1,15 +1,23 @@ -from .. import dbGIS as db -from ..decorators.restplus import api -from ..decorators.exceptions import RequestException, ParameterException, UserUnidentifiedException, \ - SnapshotNotExistingException -from ..models.user import User -from ..models.snapshots import Snapshots -from ..decorators.serializers import snapshot_load_input, snapshot_load_output, snapshot_add_input, \ - snapshot_add_output, snapshot_delete_input, snapshot_delete_output, snapshot_list_input, snapshot_list_output, \ - snapshot_update_input, snapshot_update_output from app import celery from flask_restplus import Resource + +from .. import dbGIS as db +from ..decorators.exceptions import (ParameterException, RequestException, + SnapshotNotExistingException, + UserUnidentifiedException) +from ..decorators.restplus import api +from ..decorators.serializers import (snapshot_add_input, snapshot_add_output, + snapshot_delete_input, + snapshot_delete_output, + snapshot_list_input, + snapshot_list_output, + snapshot_load_input, + snapshot_load_output, + snapshot_update_input, + snapshot_update_output) from ..decorators.timeout import return_on_timeout_endpoint +from ..models.snapshots import Snapshots +from ..models.user import User nsSnapshot = api.namespace('snapshot', description='Operations related to snapshots') ns = nsSnapshot diff --git a/api/app/api_v1/stats.py b/api/app/api_v1/stats.py index 1fe515cb..2fb77d69 100644 --- a/api/app/api_v1/stats.py +++ b/api/app/api_v1/stats.py @@ -1,41 +1,43 @@ -from app import celery +import json import logging -import re import os.path -import pandas as pd -import numpy as np -from osgeo import gdal - -from flask_restplus import Resource -from app.decorators.serializers import stats_layers_hectares_output,\ - stats_layers_nuts_input, stats_layers_nuts_output,\ - stats_layers_hectares_input, stats_list_nuts_input, stats_list_label_dataset,stats_layer_personnal_layer_input -from app.decorators.restplus import api -from app.decorators.exceptions import HugeRequestException, IntersectionException, NotEnoughPointsException, ParameterException, RequestException -from ..models.user import User +import re -from app.models.statsQueries import ElectricityMix -from app.models.statsQueries import LayersStats -from app.api_v1.upload import Uploads +import pandas as pd -from app.models.indicators import layersData +import app +import numpy as np import shapely.geometry as shapely_geom +from app import celery, model +from app.api_v1.upload import Uploads +from app.decorators.exceptions import (HugeRequestException, + IntersectionException, + NotEnoughPointsException, + ParameterException, RequestException) +from app.decorators.restplus import api +from app.decorators.serializers import (stats_layer_personnal_layer_input, + stats_layers_hectares_input, + stats_layers_hectares_output, + stats_layers_nuts_input, + stats_layers_nuts_output, + stats_list_label_dataset, + stats_list_nuts_input) +from app.helper import (adapt_layers_list, adapt_nuts_list, area_to_geom, + createAllLayers, find_key_in_dict, generate_csv_name, + generate_geotif_name, get_result_formatted, + getTypeScale, getValuesFromName, layers_filter, + projection_4326_to_3035, removeScaleLayers, + retrieveCrossIndicator, write_wkt_csv) +from app.model import check_table_existe, prepare_clip_personal_layer +from app.models import generalData, indicators +from app.models.indicators import HEATDEMAND_FACTOR, layersData +from app.models.statsQueries import ElectricityMix, LayersStats +from flask_restplus import Resource +from osgeo import gdal from .. import constants - -from app.models import generalData, indicators -from app.models.indicators import HEATDEMAND_FACTOR -from app.helper import find_key_in_dict, getValuesFromName, retrieveCrossIndicator, createAllLayers,\ - getTypeScale, adapt_layers_list, adapt_nuts_list, removeScaleLayers, layers_filter, getTypeScale, get_result_formatted, generate_geotif_name, area_to_geom, \ - write_wkt_csv, generate_csv_name,projection_4326_to_3035 -import app -import json -from app.model import check_table_existe, prepare_clip_personal_layer -from app import model from ..decorators.timeout import return_on_timeout_endpoint - - - +from ..models.user import User log = logging.getLogger(__name__) @@ -48,52 +50,51 @@ @api.response(530, 'Request Error') @api.response(531, 'Missing parameter.') class StatsLayersNutsInArea(Resource): - @return_on_timeout_endpoint() - @api.marshal_with(stats_layers_nuts_output) - @api.expect(stats_layers_nuts_input) - def post(self): - """ - Returns the statistics for specific layers, area and year - :return: - """ - #try: - # Entries - wrong_parameter = []; - try: - year = api.payload['year'] - except: - wrong_parameter.append('year') - try: - layersPayload = api.payload['layers'] - except: - wrong_parameter.append('layers') - try: - nuts = api.payload['nuts'] - except: - wrong_parameter.append('nuts') - # raise exception if parameters are false - if len(wrong_parameter) > 0: - exception_message = '' - for i in range(len(wrong_parameter)): - exception_message += wrong_parameter[i] - if i != len(wrong_parameter) - 1: - exception_message += ', ' - raise ParameterException(exception_message + '') - - # Stop execution if layers list or nuts list is empty - if not layersPayload or not nuts: - return - - # Get type - - - output, noDataLayers = LayersStats.run_stat(api.payload) - # output - return { - "layers": output, - "no_data_layers": noDataLayers, - "no_table_layers": noDataLayers - } + @return_on_timeout_endpoint() + @api.marshal_with(stats_layers_nuts_output) + @api.expect(stats_layers_nuts_input) + def post(self): + """ + Returns the statistics for specific layers, area and year + :return: + """ + #try: + # Entries + wrong_parameter = []; + try: + year = api.payload['year'] + except: + wrong_parameter.append('year') + try: + layersPayload = api.payload['layers'] + except: + wrong_parameter.append('layers') + try: + nuts = api.payload['nuts'] + except: + wrong_parameter.append('nuts') + # raise exception if parameters are false + if len(wrong_parameter) > 0: + exception_message = '' + for i in range(len(wrong_parameter)): + exception_message += wrong_parameter[i] + if i != len(wrong_parameter) - 1: + exception_message += ', ' + raise ParameterException(exception_message + '') + + # Stop execution if layers list or nuts list is empty + if not layersPayload or not nuts: + return + + # Get type + output, noDataLayers = LayersStats.run_stat(api.payload) + + # output + return { + "layers": output, + "no_data_layers": noDataLayers, + "no_table_layers": noDataLayers + } @ns.route('/layers/hectares') @@ -104,70 +105,65 @@ def post(self): @api.response(533, 'SQL error.') #@api.response(534, 'Not enough points error.') class StatsLayersHectareMulti(Resource): - @return_on_timeout_endpoint() - @api.marshal_with(stats_layers_hectares_output) - @api.expect(stats_layers_hectares_input) - def post(self): - """ - Returns the statistics for specific layers, hectares and year - :return: - """ - #try: - # Entries - wrong_parameter = []; - layersPayload = api.payload['layers'] - try: - year = api.payload['year'] - except: - wrong_parameter.append('year') - try: - layersPayload = api.payload['layers'] - except: - wrong_parameter.append('layers') - try: - areas = api.payload['areas'] - for test_area in areas: - try: - for test_point in test_area['points']: - try: - test_lng = test_point['lng'] - except: - wrong_parameter.append('lng') - try: - test_lat = test_point['lat'] - except: - wrong_parameter.append('lat') - except: - wrong_parameter.append('points') - except: - wrong_parameter.append('areas') - # raise exception if parameters are false - if len(wrong_parameter) > 0: - exception_message = '' - for i in range(len(wrong_parameter)): - exception_message += wrong_parameter[i] - if (i != len(wrong_parameter) - 1): - exception_message += ', ' - raise ParameterException(str(exception_message)) - - - - - - output, noDataLayers = LayersStats.run_stat(api.payload) - #print ("output hectare ",output) - - #output - return { - "layers": output, - "no_data_layers": noDataLayers, - "no_table_layers": noDataLayers - } - - - - - + @return_on_timeout_endpoint() + @api.marshal_with(stats_layers_hectares_output) + @api.expect(stats_layers_hectares_input) + def post(self): + """ + Returns the statistics for specific layers, hectares and year + :return: + """ + #try: + # Entries + wrong_parameter = []; + layersPayload = api.payload['layers'] + try: + year = api.payload['year'] + except: + wrong_parameter.append('year') + try: + layersPayload = api.payload['layers'] + except: + wrong_parameter.append('layers') + try: + areas = api.payload['areas'] + for test_area in areas: + try: + for test_point in test_area['points']: + try: + test_lng = test_point['lng'] + except: + wrong_parameter.append('lng') + try: + test_lat = test_point['lat'] + except: + wrong_parameter.append('lat') + except: + wrong_parameter.append('points') + except: + wrong_parameter.append('areas') + # raise exception if parameters are false + if len(wrong_parameter) > 0: + exception_message = '' + for i in range(len(wrong_parameter)): + exception_message += wrong_parameter[i] + if (i != len(wrong_parameter) - 1): + exception_message += ', ' + raise ParameterException(str(exception_message)) + + + + + + output, noDataLayers = LayersStats.run_stat(api.payload) + #print ("output hectare ",output) + + #output + return { + "layers": output, + "no_data_layers": noDataLayers, + "no_table_layers": noDataLayers + } @ns.route('/energy-mix/nuts-lau') @@ -176,175 +172,175 @@ def post(self): @api.response(530, 'Request error.') @api.response(531, 'Missing parameter.') class StatsLayersNutsInArea(Resource): - @return_on_timeout_endpoint() - @api.marshal_with(stats_list_label_dataset) - @api.expect(stats_list_nuts_input) - def post(self): - """ - Returns the statistics for specific layers, area and year - :return: - """ - # Entries - wrong_parameter = [] - try: - nuts = api.payload['nuts'] - except: - wrong_parameter.append('nuts') - - # raise exception if parameters are false - if len(wrong_parameter) > 0: - exception_message = '' - for i in range(len(wrong_parameter)): - exception_message += wrong_parameter[i] - if (i != len(wrong_parameter) - 1): - exception_message += ', ' - raise ParameterException(str(exception_message)) - - res = ElectricityMix.getEnergyMixNutsLau(adapt_nuts_list(nuts)) - return res - - - # Remove scale for each layer + @return_on_timeout_endpoint() + @api.marshal_with(stats_list_label_dataset) + @api.expect(stats_list_nuts_input) + def post(self): + """ + Returns the statistics for specific layers, area and year + :return: + """ + # Entries + wrong_parameter = [] + try: + nuts = api.payload['nuts'] + except: + wrong_parameter.append('nuts') + + # raise exception if parameters are false + if len(wrong_parameter) > 0: + exception_message = '' + for i in range(len(wrong_parameter)): + exception_message += wrong_parameter[i] + if (i != len(wrong_parameter) - 1): + exception_message += ', ' + raise ParameterException(str(exception_message)) + + res = ElectricityMix.getEnergyMixNutsLau(adapt_nuts_list(nuts)) + return res + + + # Remove scale for each layer @ns.route('/personnal-layers') class StatsPersonalLayers(Resource): - @return_on_timeout_endpoint() - @api.marshal_with(stats_layers_nuts_output) - @api.expect(stats_layer_personnal_layer_input) - def post(self): - noDataLayer=[] - result=[] - areas = api.payload['areas'] - - # if api.payload['scale_level'] == 'hectare': - # areas = area_to_geom(api.payload['areas']) - # cutline_input = write_wkt_csv(generate_csv_name(constants.UPLOAD_DIRECTORY), areas) # TODO: Projection to 3035 if raster - # else: - # cutline_input = model.get_shapefile_from_selection(api.payload['scale_level'], areas, constants.UPLOAD_DIRECTORY, '4326') - for pay in api.payload['layers']: - values=[] - data_file_name="" - token = pay['user_token'] - layer_id = pay['id'] - layer_type = pay['layer_id'] - layer_name = pay['layer_name'] - user = User.verify_auth_token(token) - upload = Uploads.query.filter_by(id=layer_id).first() - - upload_url = upload.url - if layer_name.endswith('.tif'): - cutline_input = model.get_cutline_input(areas, api.payload['scale_level'], 'raster') - filename_tif = generate_geotif_name(constants.UPLOAD_DIRECTORY) - args = app.helper.commands_in_array("gdalwarp -dstnodata 0 -cutline {} -crop_to_cutline -of GTiff {} {} -tr 100 100 -co COMPRESS=DEFLATE".format(cutline_input, upload_url, filename_tif)) - app.helper.run_command(args) - if os.path.isfile(filename_tif): - ds = gdal.Open(filename_tif) - arr = ds.GetRasterBand(1).ReadAsArray() - df = pd.DataFrame(arr) - else: - noDataLayer.append(layer_name) - continue - values = self.set_indicators_in_array(df, layer_type) - elif layer_name.endswith('.csv'): - cutline_input = model.get_cutline_input(areas, api.payload['scale_level'], 'vector') - - geojson = str(upload_url[:-3]) + "json" - if os.path.isfile(geojson): - # take geojson file instead (csv can not be clip), TODO: this on "prepare_clip_personal_layer" function? - upload_url = geojson - - cmd_cutline, output_csv = prepare_clip_personal_layer(cutline_input, upload_url) - app.helper.run_command(app.helper.commands_in_array(cmd_cutline)) - if os.path.isfile(output_csv): - df = pd.read_csv(output_csv) - if api.payload['scale_level'] != constants.hectare_name.lower() and "code" in df: - # Cannot clip with multipoliygons, TODO: no need to cut the csv with a shapefile for this - df = df[df["code"].isin(areas)] - - for ind in indicators.layersData[layer_type]['indicators']: - try: - value = df[ind['table_column']].sum() - if "agg_method" in ind and ind["agg_method"] == "mean": - value /= len(areas) - - if 'factor' in ind: # Decimal * float => rise error - value = float(value) * float(ind['factor']) - - values.append(get_result_formatted(layer_type+"_"+ind['table_column'], str(value), ind['unit'])) - except: - noDataLayer.append(layer_name) - else: - noDataLayer.append(layer_name) - continue - else: - noDataLayer.append(layer_name) - continue - - result.append({ - 'name': layer_name, - 'values': values - }) - - return { - "layers": result, - "no_data_layers": noDataLayer, - "no_table_layers": noDataLayer - } - - @staticmethod - def set_indicators_in_array(df, layer_name): - values=[] - # Set result in variables - df=df[df!=0] - counted_cells = df.count().sum() - sum_tif = 0 - min_tif = 0 - max_tif = 0 - density_tif = 0 - if counted_cells != 0: - sum_tif = df.sum().sum() - min_tif = df.min().min() - max_tif = df.max().max() - density_tif = sum_tif/counted_cells - #print(max_tif,counted_cells,min_tif,max_tif) - # Assign indicator to results - values.append(get_indicators_from_result('sum', layer_name, sum_tif)) - values.append(get_indicators_from_result('count', layer_name, counted_cells)) - values.append(get_indicators_from_result('min', layer_name, min_tif)) - values.append(get_indicators_from_result('max', layer_name, max_tif)) - values.append(get_indicators_from_result('mean', layer_name, density_tif)) - return values + @return_on_timeout_endpoint() + @api.marshal_with(stats_layers_nuts_output) + @api.expect(stats_layer_personnal_layer_input) + def post(self): + noDataLayer=[] + result=[] + areas = api.payload['areas'] + + # if api.payload['scale_level'] == 'hectare': + # areas = area_to_geom(api.payload['areas']) + # cutline_input = write_wkt_csv(generate_csv_name(constants.UPLOAD_DIRECTORY), areas) # TODO: Projection to 3035 if raster + # else: + # cutline_input = model.get_shapefile_from_selection(api.payload['scale_level'], areas, constants.UPLOAD_DIRECTORY, '4326') + for pay in api.payload['layers']: + values=[] + data_file_name="" + token = pay['user_token'] + layer_id = pay['id'] + layer_type = pay['layer_id'] + layer_name = pay['layer_name'] + user = User.verify_auth_token(token) + upload = Uploads.query.filter_by(id=layer_id).first() + + upload_url = upload.url + if layer_name.endswith('.tif'): + cutline_input = model.get_cutline_input(areas, api.payload['scale_level'], 'raster') + filename_tif = generate_geotif_name(constants.UPLOAD_DIRECTORY) + args = app.helper.commands_in_array("gdalwarp -dstnodata 0 -cutline {} -crop_to_cutline -of GTiff {} {} -tr 100 100 -co COMPRESS=DEFLATE".format(cutline_input, upload_url, filename_tif)) + app.helper.run_command(args) + if os.path.isfile(filename_tif): + ds = gdal.Open(filename_tif) + arr = ds.GetRasterBand(1).ReadAsArray() + df = pd.DataFrame(arr) + else: + noDataLayer.append(layer_name) + continue + values = self.set_indicators_in_array(df, layer_type) + elif layer_name.endswith('.csv'): + cutline_input = model.get_cutline_input(areas, api.payload['scale_level'], 'vector') + + geojson = str(upload_url[:-3]) + "json" + if os.path.isfile(geojson): + # take geojson file instead (csv can not be clip), TODO: this on "prepare_clip_personal_layer" function? + upload_url = geojson + + cmd_cutline, output_csv = prepare_clip_personal_layer(cutline_input, upload_url) + app.helper.run_command(app.helper.commands_in_array(cmd_cutline)) + if os.path.isfile(output_csv): + df = pd.read_csv(output_csv) + if api.payload['scale_level'] != constants.hectare_name.lower() and "code" in df: + # Cannot clip with multipoliygons, TODO: no need to cut the csv with a shapefile for this + df = df[df["code"].isin(areas)] + + for ind in indicators.layersData[layer_type]['indicators']: + try: + value = df[ind['table_column']].sum() + if "agg_method" in ind and ind["agg_method"] == "mean": + value /= len(areas) + + if 'factor' in ind: # Decimal * float => rise error + value = float(value) * float(ind['factor']) + + values.append(get_result_formatted(layer_type+"_"+ind['table_column'], str(value), ind['unit'])) + except: + noDataLayer.append(layer_name) + else: + noDataLayer.append(layer_name) + continue + else: + noDataLayer.append(layer_name) + continue + + result.append({ + 'name': layer_name, + 'values': values + }) + + return { + "layers": result, + "no_data_layers": noDataLayer, + "no_table_layers": noDataLayer + } + + @staticmethod + def set_indicators_in_array(df, layer_name): + values=[] + # Set result in variables + df=df[df!=0] + counted_cells = df.count().sum() + sum_tif = 0 + min_tif = 0 + max_tif = 0 + density_tif = 0 + if counted_cells != 0: + sum_tif = df.sum().sum() + min_tif = df.min().min() + max_tif = df.max().max() + density_tif = sum_tif/counted_cells + #print(max_tif,counted_cells,min_tif,max_tif) + # Assign indicator to results + values.append(get_indicators_from_result('sum', layer_name, sum_tif)) + values.append(get_indicators_from_result('count', layer_name, counted_cells)) + values.append(get_indicators_from_result('min', layer_name, min_tif)) + values.append(get_indicators_from_result('max', layer_name, max_tif)) + values.append(get_indicators_from_result('mean', layer_name, density_tif)) + return values def get_indicators_from_result(id,layer,result): - filtered_indicators = list(filter(lambda x: 'table_column' in x and x['table_column'] == id, indicators.layersData[layer]['indicators'])) - unit = layer + '_unit_' + id - value = result - name = layer + '_' + id - if len(filtered_indicators)>=1: - name = layer + '_' + filtered_indicators[0]['indicator_id'] - if 'unit' in filtered_indicators[0]: unit = filtered_indicators[0]['unit'] - if 'factor' in filtered_indicators[0]: value = result*filtered_indicators[0]['factor'] - return get_result_formatted(name,str(value),unit) - + filtered_indicators = list(filter(lambda x: 'table_column' in x and x['table_column'] == id, indicators.layersData[layer]['indicators'])) + unit = layer + '_unit_' + id + value = result + name = layer + '_' + id + if len(filtered_indicators)>=1: + name = layer + '_' + filtered_indicators[0]['indicator_id'] + if 'unit' in filtered_indicators[0]: unit = filtered_indicators[0]['unit'] + if 'factor' in filtered_indicators[0]: value = result*filtered_indicators[0]['factor'] + return get_result_formatted(name,str(value),unit) + def get_unit(id, layer): - filtered_indicators = list(filter(lambda x: 'table_column' in x and x['table_column'] == id,indicators.layersData[layer]['indicators'])) - try: - return list(filter(lambda x: 'table_column' in x and x['table_column'] == id,indicators.layersData[layer]['indicators']))[0]['unit'] - except IndexError: - return layer + '_unit_' + id + filtered_indicators = list(filter(lambda x: 'table_column' in x and x['table_column'] == id,indicators.layersData[layer]['indicators'])) + try: + return list(filter(lambda x: 'table_column' in x and x['table_column'] == id,indicators.layersData[layer]['indicators']))[0]['unit'] + except IndexError: + return layer + '_unit_' + id def get_businness_id(id, layer): - filtered_indicators = list(filter(lambda x: 'table_column' in x and x['table_column'] == id,indicators.layersData[layer]['indicators'])) - try: - return layer + '_' + filtered_indicators[0]['indicator_id'] - except IndexError: - return layer + '_' + id + filtered_indicators = list(filter(lambda x: 'table_column' in x and x['table_column'] == id,indicators.layersData[layer]['indicators'])) + try: + return layer + '_' + filtered_indicators[0]['indicator_id'] + except IndexError: + return layer + '_' + id @celery.task(name = 'energy_mix_nuts_lau') def processGenerationMix(nuts): - if not nuts: - return - res = ElectricityMix.getEnergyMixNutsLau(adapt_nuts_list(nuts)) + if not nuts: + return + res = ElectricityMix.getEnergyMixNutsLau(adapt_nuts_list(nuts)) - return res + return res diff --git a/api/app/api_v1/upload.py b/api/app/api_v1/upload.py index f4231987..d7b7caee 100644 --- a/api/app/api_v1/upload.py +++ b/api/app/api_v1/upload.py @@ -6,24 +6,34 @@ import shapely.geometry as shapely_geom from app import celery -from app.constants import USER_UPLOAD_FOLDER, UPLOAD_BASE_NAME, UPLOAD_DIRECTORY, NUTS_YEAR, LAU_YEAR +from app.constants import (LAU_YEAR, NUTS_YEAR, UPLOAD_BASE_NAME, + UPLOAD_DIRECTORY, USER_UPLOAD_FOLDER) from flask import send_file from flask_restplus import Resource from .. import dbGIS as db from ..decorators.parsers import file_upload -from ..decorators.restplus import UserUnidentifiedException, ParameterException, RequestException, \ - UserDoesntOwnUploadsException, UploadNotExistingException, \ - HugeRequestException, NotEnoughPointsException, UploadFileNotExistingException -from ..decorators.restplus import api -from ..decorators.serializers import upload_add_output, upload_list_input, upload_list_output, upload_delete_input, \ - upload_delete_output, upload_export_csv_nuts_input, upload_export_csv_hectare_input, \ - upload_export_raster_nuts_input, upload_export_raster_hectare_input, upload_download_input, \ - upload_export_cm_layer_input -from ..model import get_csv_from_nuts, get_csv_from_hectare -from ..models.uploads import Uploads, generate_tiles, allowed_file, generate_geojson, calculate_total_space -from ..models.user import User +from ..decorators.restplus import (HugeRequestException, + NotEnoughPointsException, + ParameterException, RequestException, + UploadFileNotExistingException, + UploadNotExistingException, + UserDoesntOwnUploadsException, + UserUnidentifiedException, api) +from ..decorators.serializers import (upload_add_output, upload_delete_input, + upload_delete_output, + upload_download_input, + upload_export_cm_layer_input, + upload_export_csv_hectare_input, + upload_export_csv_nuts_input, + upload_export_raster_hectare_input, + upload_export_raster_nuts_input, + upload_list_input, upload_list_output) from ..decorators.timeout import return_on_timeout_endpoint +from ..model import get_csv_from_hectare, get_csv_from_nuts +from ..models.uploads import (Uploads, allowed_file, calculate_total_space, + generate_geojson, generate_tiles) +from ..models.user import User nsUpload = api.namespace('upload', description='Operations related to file upload') ns = nsUpload @@ -740,5 +750,3 @@ def post(self=None): mimetype=mimetype, attachment_filename=upload.name + extension, as_attachment=True) - - diff --git a/api/app/api_v1/users.py b/api/app/api_v1/users.py index dba82b89..fc6f151a 100644 --- a/api/app/api_v1/users.py +++ b/api/app/api_v1/users.py @@ -2,31 +2,45 @@ import os import uuid +from app import celery +from flask import current_app, request from flask_mail import Message from flask_restplus import Resource from flask_security import SQLAlchemySessionUserDatastore -from itsdangerous import (URLSafeTimedSerializer, BadSignature, SignatureExpired) -from flask import request, current_app +from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer from passlib.hash import bcrypt -from app import celery -from .upload import calculate_total_space -from .. import constants, mail, login_manager -from ..decorators.exceptions import ParameterException, RequestException, ActivationException, \ - UserExistingException, WrongCredentialException, UserUnidentifiedException, \ - UserNotActivatedException -from ..decorators.restplus import api -from ..decorators.serializers import user_register_input, user_register_output, user_activate_input, \ - user_activate_output, user_deletion_input, user_deletion_output, user_ask_recovery_input, \ - user_ask_recovery_output, user_recovery_output, user_recovery_input, user_login_input, user_login_output, \ - user_logout_input, user_logout_output, user_profile_input, user_profile_output, user_get_information_output, \ - user_get_information_input, upload_space_used_output, upload_space_used_input, feedback_output +from .. import constants from .. import dbGIS as db -from ..models.user import User -from ..models.role import Role +from .. import login_manager, mail +from ..decorators.exceptions import (ActivationException, ParameterException, + RequestException, UserExistingException, + UserNotActivatedException, + UserUnidentifiedException, + WrongCredentialException) from ..decorators.parsers import file_upload_feedback +from ..decorators.restplus import api +from ..decorators.serializers import (feedback_output, upload_space_used_input, + upload_space_used_output, + user_activate_input, + user_activate_output, + user_ask_recovery_input, + user_ask_recovery_output, + user_deletion_input, + user_deletion_output, + user_get_information_input, + user_get_information_output, + user_login_input, user_login_output, + user_logout_input, user_logout_output, + user_profile_input, user_profile_output, + user_recovery_input, + user_recovery_output, + user_register_input, + user_register_output) from ..decorators.timeout import return_on_timeout_endpoint - +from ..models.role import Role +from ..models.user import User +from .upload import calculate_total_space # Setup Flask-Security user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) @@ -45,9 +59,9 @@ class AskingPasswordRecovery(Resource): @celery.task(name='ask for password recovery') def post(self): """ - Method to ask for a Password recovery - :return: - """ + Method to ask for a Password recovery + :return: + """ # Entries try: email = api.payload['email'] @@ -90,9 +104,9 @@ class RecoverPassword(Resource): @celery.task(name='method for recover of password') def post(self): """ - Method to recover the password - :return: - """ + Method to recover the password + :return: + """ # Entries wrong_parameter = [] try: @@ -145,9 +159,9 @@ class UserRegistering(Resource): @celery.task(name='user registration') def post(self): """ - Returns the statistics for specific layers, area and year - :return: - """ + Returns the statistics for specific layers, area and year + :return: + """ # Entries wrong_parameter = [] try: @@ -222,9 +236,9 @@ class ActivateUser(Resource): @celery.task(name='user activation') def post(self): ''' - The method called to activate a user with a token given by email - :return: - ''' + The method called to activate a user with a token given by email + :return: + ''' # Entries try: token = api.payload['token'] @@ -260,9 +274,9 @@ class DeleteUser(Resource): @celery.task(name='user deletion') def delete(self): ''' - The method called to delete a user with a token given by email - :return: - ''' + The method called to delete a user with a token given by email + :return: + ''' # Entries try: token = api.payload['token'] @@ -299,9 +313,9 @@ class LoginUser(Resource): @celery.task(name='user login') def post(self): ''' - The method called to login a user - :return: - ''' + The method called to login a user + :return: + ''' # Entries wrong_parameter = [] try: @@ -354,9 +368,9 @@ class LogoutUser(Resource): @celery.task(name='user logout') def post(self): ''' - The method called to logout a user - :return: - ''' + The method called to logout a user + :return: + ''' try: token = api.payload['token'] except: @@ -509,9 +523,9 @@ def post(self): def generate_confirmation_token(email): """ - this method will generate a confirmation token - :return: the confirmation token - """ + this method will generate a confirmation token + :return: the confirmation token + """ s = URLSafeTimedSerializer(constants.FLASK_SECRET_KEY, salt=constants.FLASK_SALT) token = s.dumps( { @@ -526,11 +540,11 @@ def generate_confirmation_token(email): def confirm_token(token, expiration=3600): ''' - This method will confirm that the given token is correct - :param token: - :param expiration: - :return: - ''' + This method will confirm that the given token is correct + :param token: + :param expiration: + :return: + ''' s = URLSafeTimedSerializer(constants.FLASK_SECRET_KEY, salt=constants.FLASK_SALT) try: data = s.loads(token) @@ -547,10 +561,10 @@ def confirm_token(token, expiration=3600): @login_manager.user_loader def load_user(user_id): ''' - this method will return the current user - :param user_id: - :return: - ''' + this method will return the current user + :param user_id: + :return: + ''' return User.query.filter_by(id=user_id).first() @@ -580,7 +594,7 @@ def post(self): Feedback type : {}
\ Feedback priority : {}
\ Description :

{}

".format(title, datetime.date.today(), name,company,feedback_type,feedback_priority, description) - + if 'file' in args and args['file'] is not None: file=args['file'] @@ -594,7 +608,7 @@ def post(self): self.send_async_mail(msg) except Exception as e: raise RequestException(str(e)) - + return { 'message':'Your feedback has been sent successfully. It will be examined and processed as soon as possible' } @celery.task(name='send mail feedback') diff --git a/api/app/bll/csv_file.py b/api/app/bll/csv_file.py index 58bece46..e69de29b 100644 --- a/api/app/bll/csv_file.py +++ b/api/app/bll/csv_file.py @@ -1,3 +0,0 @@ - - - diff --git a/api/app/constants.py b/api/app/constants.py index f57beda5..4e31a74e 100644 --- a/api/app/constants.py +++ b/api/app/constants.py @@ -135,4 +135,4 @@ DATASET_DIRECTORY = '/var/hotmaps/repositories/' NUTS_YEAR = "2013" -LAU_YEAR = NUTS_YEAR \ No newline at end of file +LAU_YEAR = NUTS_YEAR diff --git a/api/app/decorators/__init__.py b/api/app/decorators/__init__.py index 57e1aca6..a610fd37 100644 --- a/api/app/decorators/__init__.py +++ b/api/app/decorators/__init__.py @@ -1,2 +1,2 @@ -from .caching import cache_control, no_cache, etag +from .caching import cache_control, etag, no_cache diff --git a/api/app/decorators/caching.py b/api/app/decorators/caching.py index 03a0f26c..58447c89 100644 --- a/api/app/decorators/caching.py +++ b/api/app/decorators/caching.py @@ -1,6 +1,7 @@ import functools import hashlib -from flask import request, make_response, jsonify + +from flask import jsonify, make_response, request def cache_control(*directives): diff --git a/api/app/decorators/exceptions.py b/api/app/decorators/exceptions.py index 09f1bccd..f260d5e4 100644 --- a/api/app/decorators/exceptions.py +++ b/api/app/decorators/exceptions.py @@ -121,4 +121,3 @@ class ValidationError(ValueError): class ComputationalModuleError(ValueError): pass - diff --git a/api/app/decorators/json.py b/api/app/decorators/json.py index b00de39d..db06bbae 100644 --- a/api/app/decorators/json.py +++ b/api/app/decorators/json.py @@ -1,4 +1,5 @@ import functools + from flask import jsonify diff --git a/api/app/decorators/paginate.py b/api/app/decorators/paginate.py index 1c2845b0..8f67384e 100644 --- a/api/app/decorators/paginate.py +++ b/api/app/decorators/paginate.py @@ -1,5 +1,6 @@ import functools -from flask import url_for, request + +from flask import request, url_for def paginate(collection, max_per_page=25): diff --git a/api/app/decorators/parsers.py b/api/app/decorators/parsers.py index a60df621..5230a4a6 100644 --- a/api/app/decorators/parsers.py +++ b/api/app/decorators/parsers.py @@ -1,5 +1,5 @@ -from flask_restplus import reqparse import werkzeug +from flask_restplus import reqparse pagination_arguments = reqparse.RequestParser() pagination_arguments.add_argument('page', type=int, required=False, default=1, help='Page number') diff --git a/api/app/decorators/restplus.py b/api/app/decorators/restplus.py index ed391143..431ce029 100644 --- a/api/app/decorators/restplus.py +++ b/api/app/decorators/restplus.py @@ -1,14 +1,24 @@ -import traceback import logging +import traceback from flask_restplus import Api -from .. import constants from sqlalchemy.orm.exc import NoResultFound -from ..decorators.exceptions import HugeRequestException, IntersectionException, NotEnoughPointsException, \ - ParameterException, RequestException, ActivationException, UserExistingException, \ - WrongCredentialException, UserUnidentifiedException, UserDoesntOwnUploadsException, NotEnoughSpaceException, \ - UploadNotExistingException, UserNotActivatedException, SnapshotNotExistingException, \ - UploadFileNotExistingException, TimeOutException + +from .. import constants +from ..decorators.exceptions import (ActivationException, HugeRequestException, + IntersectionException, + NotEnoughPointsException, + NotEnoughSpaceException, + ParameterException, RequestException, + SnapshotNotExistingException, + TimeOutException, + UploadFileNotExistingException, + UploadNotExistingException, + UserDoesntOwnUploadsException, + UserExistingException, + UserNotActivatedException, + UserUnidentifiedException, + WrongCredentialException) log = logging.getLogger(__name__) @@ -334,4 +344,3 @@ def default_error_handler(e): def database_not_found_error_handler(e): log.warning(traceback.format_exc()) return {'message': 'A models result was required but none was found.'}, 404 - diff --git a/api/app/decorators/serializers.py b/api/app/decorators/serializers.py index 8f8a88e6..300c4f8c 100644 --- a/api/app/decorators/serializers.py +++ b/api/app/decorators/serializers.py @@ -1,8 +1,9 @@ -from flask_restplus import fields from app.decorators.restplus import api +from flask_restplus import fields from geoalchemy2.shape import to_shape from geojson import Feature, FeatureCollection, dumps + class Geometry(fields.Raw): def format(self, value): shape = to_shape(value) diff --git a/api/app/decorators/timeout.py b/api/app/decorators/timeout.py index e724a813..d0cd7f85 100644 --- a/api/app/decorators/timeout.py +++ b/api/app/decorators/timeout.py @@ -1,7 +1,8 @@ -from .exceptions import TimeOutException -from .restplus import handle_timeout_reached import signal + from ..constants import DEFAULT_TIMEOUT +from .exceptions import TimeOutException +from .restplus import handle_timeout_reached def timeout_signal_handler(signum, frame): @@ -31,4 +32,4 @@ def applicator(*args, **kwargs): return applicator - return decorate \ No newline at end of file + return decorate diff --git a/api/app/helper.py b/api/app/helper.py index 4a7c1a06..0e0f0c82 100644 --- a/api/app/helper.py +++ b/api/app/helper.py @@ -1,528 +1,529 @@ -from app.constants import LAU_TABLE -from app import celery -from shlex import split -import subprocess -import json -import uuid -import shapely.geometry as shapely_geom -import ast -from osgeo import ogr -from osgeo import osr -from . import constants -import requests -from .decorators.exceptions import RequestException -import xml.etree.ElementTree as ET -import csv -import os - -from .models.indicators import MUNICIPAL_SOLID_WASTE - -class ColorMap: - ''' - This class is used to access all informations necessary to upload a .tif to the server separating it into tiles - ''' - def __init__(self, r, g, b, a, quantity): - self.r = r - self.g = g - self.b = b - self.a = a - self.quantity = quantity - - -@celery.task(name = 'Colorize') -def colorize(layer_type, grey_tif, rgb_tif): - ''' - This method is used to check the size of the file - :param layer_type: the name of the layer type chosen for the input - :param grey_tif: the url to the input file - :param rgb_tif: the path to the output file - :return: - ''' - print('colorize') - - # we want to use a unique id for the file to be sure that it will not be duplicated in case two - uuid_temp = str(uuid.uuid4()) - xml = get_style_from_geoserver(layer_type) - color_map_objects = extract_colormap(xml) - - grey2rgb_path = create_grey2rgb_txt(color_map_objects, uuid_temp) - - args_rgba = commands_in_array("gdaldem color-relief {} {} -alpha {} -co COMPRESS=LZW".format(grey_tif, grey2rgb_path, rgb_tif)) - run_command(args_rgba) - - # we delete all temp files - for fname in os.listdir('/tmp'): - if fname.startswith(uuid_temp): - os.remove(os.path.join('/tmp', fname)) - - -def get_style_from_geoserver(layer_type): - ''' - This method will get the style from the geoserver using GET method - :param layer_type: the layer type to select - :return xml: the sld style file - ''' - # TODO: change name on geoserver? - if layer_type == MUNICIPAL_SOLID_WASTE: - layer_type = "potential_municipal_solid_waste" - - url = constants.GEOSERVER_API_URL + 'styles/' + layer_type + '.sld' - result = requests.get(url) - xml = result.content - # This piece of code is temporary, this should be removed when the workspaces on geoserver are unified - if b'No such style' in xml: - # As some layer are inside workspaces, we need to specify the workspace in order to find the correct style - url = constants.GEOSERVER_API_URL + 'workspaces/hotmaps/styles/' + layer_type + '.sld' - result = requests.get(url) - xml = result.content - return xml - - -def create_grey2rgb_txt(color_map_objects, uuid_upload): - ''' - This method will create the grey2rgb.txt file in the /tmp folder in order to convert the .tif to the rgb format - :param color_map_objects: the list of ColorMap required - :param uuid_upload: the uuid in order to have a single file - :return: the file path - ''' - # create the path and the file - grey2rgb_path = '/tmp/' + uuid_upload + 'grey2rgb.txt' - grey2rgb = open(grey2rgb_path, 'w') - - # complete the file - for color_map_object in color_map_objects: - grey2rgb.write( - str(color_map_object.quantity) + " " + - str(color_map_object.r) + " " + - str(color_map_object.g) + " " + - str(color_map_object.b) + " " + - str(color_map_object.a) + "\r\n" - ) - - # close the file connection and return the path - grey2rgb.close() - return grey2rgb_path - - -def extract_colormap(xml): - ''' - This method will extract the colormap of a sld stylesheet - :param xml: the xml file - :return: an array of the different color map - ''' - # create the xml tree - try: - root = ET.fromstring(xml) - except Exception as e: - raise RequestException(str(xml)) - ns = {'sld': 'http://www.opengis.net/sld'} - # get the list of Color map - color_map_list = root.findall(".//sld:ColorMapEntry", ns) - color_map_objects = [] - # for each color map get the color, the opacity and the quantity - for color_map in color_map_list: - color_tuple = hex_to_rgb(color_map.get('color')) - opacity = int(float(color_map.get('opacity')) * 255) - quantity = color_map.get('quantity') - color_map_object = ColorMap(color_tuple[0], color_tuple[1], color_tuple[2], opacity, quantity) - - # add the color map object to the list - color_map_objects.append(color_map_object) - return color_map_objects - - -def hex_to_rgb(value): - ''' - This method is used to convert an hexadecimal into a tuple of rgb values - :param value: hexadecimal - :return: a tuple of rgb - ''' - value = value.lstrip('#') - lv = len(value) - return tuple(int(value[i:i + lv // 3], 16) for i in range(0, lv, lv // 3)) - - -def find_key_in_dict(key, dictionary): - for k, v in dictionary.items(): - if k == key: - yield v - elif isinstance(v, dict): - for result in find_key_in_dict(key, v): - yield result - elif isinstance(v, list): - for d in v: - for result in find_key_in_dict(key, d): - yield result - -def retrieveCrossIndicator(denominator_indicator_name, numerator_indicator_name, layers, payload_output): - if denominator_indicator_name in layers and numerator_indicator_name in layers: - numerator = getValuesFromName(numerator_indicator_name,payload_output) - denominator = getValuesFromName(denominator_indicator_name,payload_output) - generateCrossIndicator(numerator, denominator,numerator_indicator_name, payload_output) - -def generateCrossIndicator(numerator, denominator, value_to_append, output): - denominator_val = float(denominator.get('value', 1)) - denominator_val = denominator_val if denominator_val > 0 else 1 - numerator_val = float(numerator.get('value', 0)) - v = { - 'name': numerator['name'] + '_per_' + denominator['name'], - 'value': numerator_val / denominator_val, - 'unit': numerator.get('unit') + '/' + denominator.get('unit') - } - for x in output: - if x['name'] == value_to_append: - x['values'].append(v) - -def getValuesFromName(name, output): - values = None - for i in output: - if i['name'] == name: - values = i['values'][0] - break - return values -def unicode_array_to_string(unicode_string): - return ast.literal_eval(unicode_string) -def unicode_string_to_string(unicode_string): - return str(unicode_string).encode('ascii','ignore') - - -def test_display(value): - pass - #print ('value ', value) - #print ('type ', type(value)) -def getDictFromJson(output): - outputdumps = json.dumps(output) - outputloads = json.loads(outputdumps)[0] - return outputloads - -def roundValue(value): - return round(value, 1) - -def getGenerationMixColor(value): - switcher = { - "Nuklear": "#909090", - "Lignite": "#556B2F", - "Hard coal": "#000000", - "Natural gas": "#FFD700", - "Oil": "#8B0000", - "Other fossil fuels": "#A9A9A9", - "PV": "#FFFF00", - "Wind ": "#D8BFD8", - "Biomass": "#228B22", - "Hydro": "#1E90FF", - "No information on source": "#FFFAFA", - } - return switcher.get(value, "#D8BFD8") - - -def get_result_formatted(name='not_defined', value=0, unit='unit'): - return { - 'name': name, - 'value': value, - 'unit': unit - } - - - -def generate_geotif_name(directory): - filename = generate_file(directory, '.tif') - return filename - -def generate_shapefile_name(directory): - filename = generate_file(directory, '.shp') - return filename -def generate_csv_name(directory): - filename = generate_file(directory, '.csv') - return filename -def generate_archive(directory): - filename = generate_file(directory, '.zip') - return filename -def generate_file(directory,extension): - filename = directory+'/' + str(uuid.uuid4()) + extension - return filename - -def generate_directory_name(): - return str(uuid.uuid4()) - -def area_to_geom(areas): - polyArray = [] - # convert to polygon format for each polygon and store them in polyArray - for polygon in areas: - po = shapely_geom.Polygon([[p['lng'], p['lat']] for p in polygon['points']]) - polyArray.append(po) - # convert array of polygon into multipolygon - multipolygon = shapely_geom.MultiPolygon(polyArray) - #geom = "SRID=4326;{}".format(multipolygon.wkt) - - geom = multipolygon.wkt - return geom - -def adapt_nuts_list(nuts): - # Store nuts in new custom list - nutsPayload = [] - for n in nuts: - if n not in nutsPayload: - nutsPayload.append(str(n)) - - # Adapt format of list for the query - nutsListQuery = str(nutsPayload) - nutsListQuery = nutsListQuery[1:] # Remove the left hook - nutsListQuery = nutsListQuery[:-1] # Remove the right hook - - return nutsListQuery - -def generate_payload_for_compute(inputs_raster_selection,inputs_parameter_selection): - - data_output = {} - - data_output.update({ - - 'inputs_parameter_selection':inputs_parameter_selection - }) - data_output.update({ - - 'inputs_raster_selection':inputs_raster_selection - }) - ##print ('data_output',data_output) - data = json.dumps(data_output) - return data - -def remove_None_in_turple(tupleX): - tupleX = [x for x in tupleX if x is not None] - return tupleX - -def write_wkt_csv(output_file,content): - - with open(output_file, mode='w') as csv_file: - fieldnames = ['id', 'WKT'] - writer = csv.DictWriter(csv_file, fieldnames=fieldnames) - - writer.writeheader() - writer.writerow({'id': '1', 'WKT': content}) - return output_file - -def projection_4326_to_3035(wkt): - # use the database to transform the geometry from 3857 to 4326 - source = osr.SpatialReference() - source.ImportFromEPSG(4326) - - target = osr.SpatialReference() - target.ImportFromEPSG(3035) - - transform = osr.CoordinateTransformation(source, target) - - point = ogr.CreateGeometryFromWkt(wkt) - point.Transform(transform) - - return point.ExportToWkt() - - - -def zipdir(path, ziph): - # ziph is zipfile handle - for root, dirs, files in os.walk(path): - for file in files: - ziph.write(os.path.join(root, file)) - -def retrieve_list_from_sql_result(results): - response = [] - for value in results: - ##print ('value', value) - ze_value = {} - i = 0 - for key in results.description: - ze_value[key[0]]= str(value[i]) - if isinstance(unicode_string_to_string(value[i]), str): - val = unicode_string_to_string(value[i]) - if val.find('[') == 0: # and value.find(']')== - #print ('value ', val) - ze_value[key[0]]= unicode_array_to_string(value[i]) - elif isinstance(value[i], str): - val = value[i] - if val.find('[') == 0: # and value.find(']')== - #print ('value ', val) - ze_value[key[0]]= unicode_array_to_string(value[i]) - i = i + 1 - response.append(ze_value) - return response - -def force_decode(string, codecs=['utf8', 'cp1252']): - for i in codecs: - try: - return string.decode(i) - except UnicodeDecodeError: - pass -def from_dict_to_unique_array(results,key): - response = [] - for value in results: - ze_value = value[key] - response.append(ze_value) - return response - -def sampling_data(listValues): - # Get number of values - numberOfValues = len(listValues) - - # Create the points for the curve with the X and Y axis - listPoints = [] - for n, l in enumerate(listValues): - listPoints.append({ - 'X':n+1, - 'Y':listValues[n] - }) - - # Sampling of the values - cut1 = int(numberOfValues*constants.POINTS_FIRST_GROUP_PERCENTAGE) - cut2 = int(cut1+(numberOfValues*constants.POINTS_SECOND_GROUP_PERCENTAGE)) - cut3 = int(cut2+(numberOfValues*constants.POINTS_THIRD_GROUP_PERCENTAGE)) - - firstGroup = listPoints[0:cut1:constants.POINTS_FIRST_GROUP_STEP] - secondGroup = listPoints[cut1:cut2:constants.POINTS_SECOND_GROUP_STEP] - thirdGroup = listPoints[cut2:cut3:constants.POINTS_THIRD_GROUP_STEP] - fourthGroup = listPoints[cut3:numberOfValues:constants.POINTS_FOURTH_GROUP_STEP] - - # Get min and max values needed for the sampling list - maxValue = min(listPoints) - minValue = max(listPoints) - - # Concatenate the groups to a new list of points (sampling list) - finalListPoints = firstGroup+secondGroup+thirdGroup+fourthGroup - - # Add max value at the beginning if the list doesn't contain it - if maxValue not in finalListPoints: - finalListPoints.insert(0, maxValue) - - # Add min value at the end if the list doesn't contain it - if minValue not in finalListPoints: - finalListPoints.append(minValue) - - return finalListPoints -def nuts_array_to_string(nuts): - nuts_transformed = ''.join("'"+str(nu)+"'," for nu in nuts)[:-1] - return nuts_transformed - -def transform_nuts_list(nuts): - # Store nuts in new custom list - nutsPayload = [] - for n in nuts: - n = n[:4] - if n not in nutsPayload: - nutsPayload.append(str(n)) - - # Adapt format of list for the query - nutsListQuery = str(nutsPayload) - nutsListQuery = nutsListQuery[1:] # Remove the left hook - nutsListQuery = nutsListQuery[:-1] # Remove the right hook - - return nutsListQuery - - -def createAllLayers(layers): - allLayers = [] - for l in layers: - allLayers.append(l) - allLayers.append(l+'_ha') - allLayers.append(l+'_nuts3') - allLayers.append(l+'_nuts2') - allLayers.append(l+'_nuts1') - allLayers.append(l+'_nuts0') - allLayers.append(l+'_lau2') - - return allLayers - -def getTypeScale(layers): - if layers: - if layers[0].endswith('lau2'): - return 'lau' - else: - return 'nuts' - else: - return '' - -def areas_to_geom(areas): - polygon_array=[] - for polygon in areas: - po = shapely_geom.Polygon([[p['lng'], p['lat']] for p in polygon['points']]) - polygon_array.append(po) - # convert array of polygon into multipolygon - mp = shapely_geom.MultiPolygon(polygon_array) - return mp.wkt - -def adapt_layers_list(layersPayload, type, allLayers): - layers = [] - if type == 'lau': - for layer in layersPayload: - if layer in allLayers: - layer = layer[:-5] # Remove the type for each layer - layers.append(layer) - elif type == 'ha': - for layer in layersPayload: - if layer in allLayers: - layer = layer[:-3] # Remove the type for each layer - layers.append(layer) - else: - for layer in layersPayload: - if layer in allLayers: - layer = layer[:-6] # Remove the type for each layer - layers.append(layer) - - return layers -def removeScaleLayers(layersList, type): - layers = [] - if type == 'lau': - for layer in layersList: - layer = layer[:-5] # Remove the type for each layer - layers.append(layer) - elif type == 'ha': - for layer in layersList: - layer = layer[:-3] # Remove the type for each layer - layers.append(layer) - else: - for layer in layersList: - layer = layer[:-6] # Remove the type for each layer - layers.append(layer) - - return layers - -def layers_filter(layersPayload, list): - layers = [] - for l in layersPayload: - if l not in list: - layers.append(l) - - return layers -def get_nuts_query_selection(nuts, scale_level_table, scale_id): - - if scale_level_table == 'nuts': - scale_schema = 'geo' - return """nutsSelection as ( - SELECT nuts.nuts_id as nuts2_id, tbl2."""+scale_id+""" as scale_id - from geo.nuts nuts, """+scale_schema+"""."""+scale_level_table+""" tbl2 - where tbl2.year = date('2013-01-01') and tbl2."""+scale_id+""" in ("""+nuts+""") - and st_within(st_transform(tbl2.geom,"""+constants.CRS_NUTS+"""),nuts.geom) - and nuts.stat_levl_ = 2 - group by nuts.nuts_id, tbl2."""+scale_id+"""),""" - - else: - scale_schema = 'public' - return """nutsSelection as ( - SELECT nuts.nuts_id as nuts2_id, tbl2."""+scale_id+""" as scale_id - from geo.nuts nuts, """+scale_schema+"""."""+LAU_TABLE+""" tbl2 - where tbl2."""+scale_id+""" in ("""+nuts+""") - and st_within(st_centroid(tbl2.geom),nuts.geom) - and nuts.stat_levl_ = 2 - group by nuts.nuts_id, tbl2."""+scale_id+"""),""" - - -def commands_in_array(com_string): - return split(com_string) - - -def run_command(arr): - process = subprocess.Popen( - arr, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - if process.wait(): - print(f"Not able to execute: {arr}\n returncode: {process.returncode}") - stdout, stderr = process.communicate() - print(f"stdout:\n{stdout}\nstderr:\n{stderr}") \ No newline at end of file +import ast +import csv +import json +import os +import subprocess +import uuid +import xml.etree.ElementTree as ET +from shlex import split + +import requests +import shapely.geometry as shapely_geom +from app import celery +from app.constants import LAU_TABLE +from osgeo import ogr, osr + +from . import constants +from .decorators.exceptions import RequestException +from .models.indicators import MUNICIPAL_SOLID_WASTE + + +class ColorMap: + ''' + This class is used to access all informations necessary to upload a .tif to the server separating it into tiles + ''' + def __init__(self, r, g, b, a, quantity): + self.r = r + self.g = g + self.b = b + self.a = a + self.quantity = quantity + + +@celery.task(name = 'Colorize') +def colorize(layer_type, grey_tif, rgb_tif): + ''' + This method is used to check the size of the file + :param layer_type: the name of the layer type chosen for the input + :param grey_tif: the url to the input file + :param rgb_tif: the path to the output file + :return: + ''' + print('colorize') + + # we want to use a unique id for the file to be sure that it will not be duplicated in case two + uuid_temp = str(uuid.uuid4()) + xml = get_style_from_geoserver(layer_type) + color_map_objects = extract_colormap(xml) + + grey2rgb_path = create_grey2rgb_txt(color_map_objects, uuid_temp) + + args_rgba = commands_in_array("gdaldem color-relief {} {} -alpha {} -co COMPRESS=LZW".format(grey_tif, grey2rgb_path, rgb_tif)) + run_command(args_rgba) + + # we delete all temp files + for fname in os.listdir('/tmp'): + if fname.startswith(uuid_temp): + os.remove(os.path.join('/tmp', fname)) + + +def get_style_from_geoserver(layer_type): + ''' + This method will get the style from the geoserver using GET method + :param layer_type: the layer type to select + :return xml: the sld style file + ''' + # TODO: change name on geoserver? + if layer_type == MUNICIPAL_SOLID_WASTE: + layer_type = "potential_municipal_solid_waste" + + url = constants.GEOSERVER_API_URL + 'styles/' + layer_type + '.sld' + result = requests.get(url) + xml = result.content + # This piece of code is temporary, this should be removed when the workspaces on geoserver are unified + if b'No such style' in xml: + # As some layer are inside workspaces, we need to specify the workspace in order to find the correct style + url = constants.GEOSERVER_API_URL + 'workspaces/hotmaps/styles/' + layer_type + '.sld' + result = requests.get(url) + xml = result.content + return xml + + +def create_grey2rgb_txt(color_map_objects, uuid_upload): + ''' + This method will create the grey2rgb.txt file in the /tmp folder in order to convert the .tif to the rgb format + :param color_map_objects: the list of ColorMap required + :param uuid_upload: the uuid in order to have a single file + :return: the file path + ''' + # create the path and the file + grey2rgb_path = '/tmp/' + uuid_upload + 'grey2rgb.txt' + grey2rgb = open(grey2rgb_path, 'w') + + # complete the file + for color_map_object in color_map_objects: + grey2rgb.write( + str(color_map_object.quantity) + " " + + str(color_map_object.r) + " " + + str(color_map_object.g) + " " + + str(color_map_object.b) + " " + + str(color_map_object.a) + "\r\n" + ) + + # close the file connection and return the path + grey2rgb.close() + return grey2rgb_path + + +def extract_colormap(xml): + ''' + This method will extract the colormap of a sld stylesheet + :param xml: the xml file + :return: an array of the different color map + ''' + # create the xml tree + try: + root = ET.fromstring(xml) + except Exception as e: + raise RequestException(str(xml)) + ns = {'sld': 'http://www.opengis.net/sld'} + # get the list of Color map + color_map_list = root.findall(".//sld:ColorMapEntry", ns) + color_map_objects = [] + # for each color map get the color, the opacity and the quantity + for color_map in color_map_list: + color_tuple = hex_to_rgb(color_map.get('color')) + opacity = int(float(color_map.get('opacity')) * 255) + quantity = color_map.get('quantity') + color_map_object = ColorMap(color_tuple[0], color_tuple[1], color_tuple[2], opacity, quantity) + + # add the color map object to the list + color_map_objects.append(color_map_object) + return color_map_objects + + +def hex_to_rgb(value): + ''' + This method is used to convert an hexadecimal into a tuple of rgb values + :param value: hexadecimal + :return: a tuple of rgb + ''' + value = value.lstrip('#') + lv = len(value) + return tuple(int(value[i:i + lv // 3], 16) for i in range(0, lv, lv // 3)) + + +def find_key_in_dict(key, dictionary): + for k, v in dictionary.items(): + if k == key: + yield v + elif isinstance(v, dict): + for result in find_key_in_dict(key, v): + yield result + elif isinstance(v, list): + for d in v: + for result in find_key_in_dict(key, d): + yield result + +def retrieveCrossIndicator(denominator_indicator_name, numerator_indicator_name, layers, payload_output): + if denominator_indicator_name in layers and numerator_indicator_name in layers: + numerator = getValuesFromName(numerator_indicator_name,payload_output) + denominator = getValuesFromName(denominator_indicator_name,payload_output) + generateCrossIndicator(numerator, denominator,numerator_indicator_name, payload_output) + +def generateCrossIndicator(numerator, denominator, value_to_append, output): + denominator_val = float(denominator.get('value', 1)) + denominator_val = denominator_val if denominator_val > 0 else 1 + numerator_val = float(numerator.get('value', 0)) + v = { + 'name': numerator['name'] + '_per_' + denominator['name'], + 'value': numerator_val / denominator_val, + 'unit': numerator.get('unit') + '/' + denominator.get('unit') + } + for x in output: + if x['name'] == value_to_append: + x['values'].append(v) + +def getValuesFromName(name, output): + values = None + for i in output: + if i['name'] == name: + values = i['values'][0] + break + return values +def unicode_array_to_string(unicode_string): + return ast.literal_eval(unicode_string) +def unicode_string_to_string(unicode_string): + return str(unicode_string).encode('ascii','ignore') + + +def test_display(value): + pass + #print ('value ', value) + #print ('type ', type(value)) +def getDictFromJson(output): + outputdumps = json.dumps(output) + outputloads = json.loads(outputdumps)[0] + return outputloads + +def roundValue(value): + return round(value, 1) + +def getGenerationMixColor(value): + switcher = { + "Nuklear": "#909090", + "Lignite": "#556B2F", + "Hard coal": "#000000", + "Natural gas": "#FFD700", + "Oil": "#8B0000", + "Other fossil fuels": "#A9A9A9", + "PV": "#FFFF00", + "Wind ": "#D8BFD8", + "Biomass": "#228B22", + "Hydro": "#1E90FF", + "No information on source": "#FFFAFA", + } + return switcher.get(value, "#D8BFD8") + + +def get_result_formatted(name='not_defined', value=0, unit='unit'): + return { + 'name': name, + 'value': value, + 'unit': unit + } + + + +def generate_geotif_name(directory): + filename = generate_file(directory, '.tif') + return filename + +def generate_shapefile_name(directory): + filename = generate_file(directory, '.shp') + return filename +def generate_csv_name(directory): + filename = generate_file(directory, '.csv') + return filename +def generate_archive(directory): + filename = generate_file(directory, '.zip') + return filename +def generate_file(directory,extension): + filename = directory+'/' + str(uuid.uuid4()) + extension + return filename + +def generate_directory_name(): + return str(uuid.uuid4()) + +def area_to_geom(areas): + polyArray = [] + # convert to polygon format for each polygon and store them in polyArray + for polygon in areas: + po = shapely_geom.Polygon([[p['lng'], p['lat']] for p in polygon['points']]) + polyArray.append(po) + # convert array of polygon into multipolygon + multipolygon = shapely_geom.MultiPolygon(polyArray) + #geom = "SRID=4326;{}".format(multipolygon.wkt) + + geom = multipolygon.wkt + return geom + +def adapt_nuts_list(nuts): + # Store nuts in new custom list + nutsPayload = [] + for n in nuts: + if n not in nutsPayload: + nutsPayload.append(str(n)) + + # Adapt format of list for the query + nutsListQuery = str(nutsPayload) + nutsListQuery = nutsListQuery[1:] # Remove the left hook + nutsListQuery = nutsListQuery[:-1] # Remove the right hook + + return nutsListQuery + +def generate_payload_for_compute(inputs_raster_selection,inputs_parameter_selection): + + data_output = {} + + data_output.update({ + + 'inputs_parameter_selection':inputs_parameter_selection + }) + data_output.update({ + + 'inputs_raster_selection':inputs_raster_selection + }) + ##print ('data_output',data_output) + data = json.dumps(data_output) + return data + +def remove_None_in_turple(tupleX): + tupleX = [x for x in tupleX if x is not None] + return tupleX + +def write_wkt_csv(output_file,content): + + with open(output_file, mode='w') as csv_file: + fieldnames = ['id', 'WKT'] + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + + writer.writeheader() + writer.writerow({'id': '1', 'WKT': content}) + return output_file + +def projection_4326_to_3035(wkt): + # use the database to transform the geometry from 3857 to 4326 + source = osr.SpatialReference() + source.ImportFromEPSG(4326) + + target = osr.SpatialReference() + target.ImportFromEPSG(3035) + + transform = osr.CoordinateTransformation(source, target) + + point = ogr.CreateGeometryFromWkt(wkt) + point.Transform(transform) + + return point.ExportToWkt() + + + +def zipdir(path, ziph): + # ziph is zipfile handle + for root, dirs, files in os.walk(path): + for file in files: + ziph.write(os.path.join(root, file)) + +def retrieve_list_from_sql_result(results): + response = [] + for value in results: + ##print ('value', value) + ze_value = {} + i = 0 + for key in results.description: + ze_value[key[0]]= str(value[i]) + if isinstance(unicode_string_to_string(value[i]), str): + val = unicode_string_to_string(value[i]) + if val.find('[') == 0: # and value.find(']')== + #print ('value ', val) + ze_value[key[0]]= unicode_array_to_string(value[i]) + elif isinstance(value[i], str): + val = value[i] + if val.find('[') == 0: # and value.find(']')== + #print ('value ', val) + ze_value[key[0]]= unicode_array_to_string(value[i]) + i = i + 1 + response.append(ze_value) + return response + +def force_decode(string, codecs=['utf8', 'cp1252']): + for i in codecs: + try: + return string.decode(i) + except UnicodeDecodeError: + pass +def from_dict_to_unique_array(results,key): + response = [] + for value in results: + ze_value = value[key] + response.append(ze_value) + return response + +def sampling_data(listValues): + # Get number of values + numberOfValues = len(listValues) + + # Create the points for the curve with the X and Y axis + listPoints = [] + for n, l in enumerate(listValues): + listPoints.append({ + 'X':n+1, + 'Y':listValues[n] + }) + + # Sampling of the values + cut1 = int(numberOfValues*constants.POINTS_FIRST_GROUP_PERCENTAGE) + cut2 = int(cut1+(numberOfValues*constants.POINTS_SECOND_GROUP_PERCENTAGE)) + cut3 = int(cut2+(numberOfValues*constants.POINTS_THIRD_GROUP_PERCENTAGE)) + + firstGroup = listPoints[0:cut1:constants.POINTS_FIRST_GROUP_STEP] + secondGroup = listPoints[cut1:cut2:constants.POINTS_SECOND_GROUP_STEP] + thirdGroup = listPoints[cut2:cut3:constants.POINTS_THIRD_GROUP_STEP] + fourthGroup = listPoints[cut3:numberOfValues:constants.POINTS_FOURTH_GROUP_STEP] + + # Get min and max values needed for the sampling list + maxValue = min(listPoints) + minValue = max(listPoints) + + # Concatenate the groups to a new list of points (sampling list) + finalListPoints = firstGroup+secondGroup+thirdGroup+fourthGroup + + # Add max value at the beginning if the list doesn't contain it + if maxValue not in finalListPoints: + finalListPoints.insert(0, maxValue) + + # Add min value at the end if the list doesn't contain it + if minValue not in finalListPoints: + finalListPoints.append(minValue) + + return finalListPoints +def nuts_array_to_string(nuts): + nuts_transformed = ''.join("'"+str(nu)+"'," for nu in nuts)[:-1] + return nuts_transformed + +def transform_nuts_list(nuts): + # Store nuts in new custom list + nutsPayload = [] + for n in nuts: + n = n[:4] + if n not in nutsPayload: + nutsPayload.append(str(n)) + + # Adapt format of list for the query + nutsListQuery = str(nutsPayload) + nutsListQuery = nutsListQuery[1:] # Remove the left hook + nutsListQuery = nutsListQuery[:-1] # Remove the right hook + + return nutsListQuery + + +def createAllLayers(layers): + allLayers = [] + for l in layers: + allLayers.append(l) + allLayers.append(l+'_ha') + allLayers.append(l+'_nuts3') + allLayers.append(l+'_nuts2') + allLayers.append(l+'_nuts1') + allLayers.append(l+'_nuts0') + allLayers.append(l+'_lau2') + + return allLayers + +def getTypeScale(layers): + if layers: + if layers[0].endswith('lau2'): + return 'lau' + else: + return 'nuts' + else: + return '' + +def areas_to_geom(areas): + polygon_array=[] + for polygon in areas: + po = shapely_geom.Polygon([[p['lng'], p['lat']] for p in polygon['points']]) + polygon_array.append(po) + # convert array of polygon into multipolygon + mp = shapely_geom.MultiPolygon(polygon_array) + return mp.wkt + +def adapt_layers_list(layersPayload, type, allLayers): + layers = [] + if type == 'lau': + for layer in layersPayload: + if layer in allLayers: + layer = layer[:-5] # Remove the type for each layer + layers.append(layer) + elif type == 'ha': + for layer in layersPayload: + if layer in allLayers: + layer = layer[:-3] # Remove the type for each layer + layers.append(layer) + else: + for layer in layersPayload: + if layer in allLayers: + layer = layer[:-6] # Remove the type for each layer + layers.append(layer) + + return layers +def removeScaleLayers(layersList, type): + layers = [] + if type == 'lau': + for layer in layersList: + layer = layer[:-5] # Remove the type for each layer + layers.append(layer) + elif type == 'ha': + for layer in layersList: + layer = layer[:-3] # Remove the type for each layer + layers.append(layer) + else: + for layer in layersList: + layer = layer[:-6] # Remove the type for each layer + layers.append(layer) + + return layers + +def layers_filter(layersPayload, list): + layers = [] + for l in layersPayload: + if l not in list: + layers.append(l) + + return layers +def get_nuts_query_selection(nuts, scale_level_table, scale_id): + + if scale_level_table == 'nuts': + scale_schema = 'geo' + return """nutsSelection as ( + SELECT nuts.nuts_id as nuts2_id, tbl2."""+scale_id+""" as scale_id + from geo.nuts nuts, """+scale_schema+"""."""+scale_level_table+""" tbl2 + where tbl2.year = date('2013-01-01') and tbl2."""+scale_id+""" in ("""+nuts+""") + and st_within(st_transform(tbl2.geom,"""+constants.CRS_NUTS+"""),nuts.geom) + and nuts.stat_levl_ = 2 + group by nuts.nuts_id, tbl2."""+scale_id+"""),""" + + else: + scale_schema = 'public' + return """nutsSelection as ( + SELECT nuts.nuts_id as nuts2_id, tbl2."""+scale_id+""" as scale_id + from geo.nuts nuts, """+scale_schema+"""."""+LAU_TABLE+""" tbl2 + where tbl2."""+scale_id+""" in ("""+nuts+""") + and st_within(st_centroid(tbl2.geom),nuts.geom) + and nuts.stat_levl_ = 2 + group by nuts.nuts_id, tbl2."""+scale_id+"""),""" + + +def commands_in_array(com_string): + return split(com_string) + + +def run_command(arr): + process = subprocess.Popen( + arr, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if process.wait(): + print(f"Not able to execute: {arr}\n returncode: {process.returncode}") + stdout, stderr = process.communicate() + print(f"stdout:\n{stdout}\nstderr:\n{stderr}") diff --git a/api/app/helper/gdal2tiles-multiprocess.py b/api/app/helper/gdal2tiles-multiprocess.py index 22df8a97..0552064a 100755 --- a/api/app/helper/gdal2tiles-multiprocess.py +++ b/api/app/helper/gdal2tiles-multiprocess.py @@ -40,7 +40,12 @@ # Hacked to make changes to the stacking order.. spruceboy@gmail.com +import math +import multiprocessing +import os import sys +import tempfile +from optparse import OptionGroup, OptionParser try: from osgeo import gdal @@ -50,8 +55,6 @@ print 'You are using "old gen" bindings. gdal2tiles needs "new gen" bindings.' sys.exit(1) -import os -import math try: from PIL import Image @@ -63,9 +66,6 @@ pass -import multiprocessing -import tempfile -from optparse import OptionParser, OptionGroup __version__ = '$Id: gdal2tiles.py 27349 2014-05-16 18:58:51Z rouault $' @@ -119,7 +119,6 @@ Class is available under the open-source GDAL license (www.gdal.org). """ -import math MAXZOOMLEVEL = 32 diff --git a/api/app/helper/gdal2tiles.py b/api/app/helper/gdal2tiles.py index b3cfcc43..0c77d9ee 100644 --- a/api/app/helper/gdal2tiles.py +++ b/api/app/helper/gdal2tiles.py @@ -37,20 +37,19 @@ # DEALINGS IN THE SOFTWARE. # ****************************************************************************** -from __future__ import print_function, division +from __future__ import division, print_function import math -from multiprocessing import Pipe, Pool, Process, Manager import os -import tempfile -import threading import shutil import sys +import tempfile +import threading +from multiprocessing import Manager, Pipe, Pool, Process from uuid import uuid4 from xml.etree import ElementTree -from osgeo import gdal -from osgeo import osr +from osgeo import gdal, osr try: from PIL import Image diff --git a/api/app/helper/gdal2tiles.py.orig b/api/app/helper/gdal2tiles.py.orig index b3cfcc43..0c77d9ee 100644 --- a/api/app/helper/gdal2tiles.py.orig +++ b/api/app/helper/gdal2tiles.py.orig @@ -37,20 +37,19 @@ # DEALINGS IN THE SOFTWARE. # ****************************************************************************** -from __future__ import print_function, division +from __future__ import division, print_function import math -from multiprocessing import Pipe, Pool, Process, Manager import os -import tempfile -import threading import shutil import sys +import tempfile +import threading +from multiprocessing import Manager, Pipe, Pool, Process from uuid import uuid4 from xml.etree import ElementTree -from osgeo import gdal -from osgeo import osr +from osgeo import gdal, osr try: from PIL import Image diff --git a/api/app/model.py b/api/app/model.py index b45f5e95..bf4729c4 100644 --- a/api/app/model.py +++ b/api/app/model.py @@ -1,746 +1,747 @@ -import uuid - -import app.helper -from app.decorators.exceptions import ValidationError, HugeRequestException, RequestException, NotEnoughPointsException - - -from .helper import area_to_geom, write_wkt_csv, generate_csv_name, projection_4326_to_3035, commands_in_array, \ - run_command - -try: - from shlex import quote -except ImportError: - from pipes import quote -import subprocess -from app.constants import DATASET_DIRECTORY, USER_DB,HOST_DB,PASSWORD_DB,PORT_DB,DATABASE_DB -from app.constants import DATASET_DIRECTORY, UPLOAD_DIRECTORY, NUTS_YEAR, LAU_YEAR -from datetime import datetime -import psycopg2 -import sqlalchemy.pool as pool -import sqlite3 -from app import celery, dbGIS as db, constants -from app.constants import CM_DB_NAME -from app import helper -from app import sql_queries -from .models.uploads import Uploads, generate_csv_string -import os -import shapely.geometry as shapely_geom -try: - import ogr -except ImportError: - from osgeo import ogr - -try: - import osr -except ImportError: - from osgeo import osr -basedir = os.path.abspath(os.path.dirname(__file__)) - -db_path = os.path.join(basedir, '../data.sqlite') - -DB_NAME = CM_DB_NAME - -def getConnection_db_CM(): - c = sqlite3.connect(DB_NAME) - return c - - -myCMpool = pool.QueuePool(getConnection_db_CM, max_overflow=10, pool_size=15) - - -def addRegisterCalulationModule(data): - """ - this request will get the signature of a CM and will insert it to the database - :param data: signature payload of the CM - :return: - """ - - cm_name = data['cm_name'] - - - - wiki_url = "" - try: - wiki_url = data['wiki_url'] - except: - pass - category = data['category'] - type_layer_needed = data['type_layer_needed'] - authorized_scale = "[]" - try: - authorized_scale = data['authorized_scale'] - except: - pass - description_link = "" - try: - description_link = data['description_link'] - except: - pass - vectors_needed = "[]" - try: - vectors_needed = data['vectors_needed'] - except: - pass - - try: - type_vectors_needed = data['type_vectors_needed'] - except: - pass - cm_description = data['cm_description'] - cm_url = data['cm_url'] - cm_Id = data['id'] - layers_needed = data['layers_needed'] - updatedAt = datetime.utcnow() - createdAt = datetime.utcnow() - conn = myCMpool.connect() - cursor = conn.cursor() - cursor.execute("INSERT INTO calculation_module (cm_id, cm_name, cm_description, category, cm_url, layers_needed, createdAt, updateAt,type_layer_needed,authorized_scale,description_link,vectors_needed,wiki_url) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", ( cm_Id, cm_name, cm_description, category, cm_url, layers_needed, createdAt, updatedAt ,type_layer_needed,authorized_scale,description_link,vectors_needed,type_vectors_needed,wiki_url)) - cursor.close() - -def init_sqlite_caculation_module_database(dbname=DB_NAME): - """ - - :param dbname: this part will manually create the database for CM - :return: conn : this is the connection to the database - """ - conn = sqlite3.connect(dbname) - cursor = conn.cursor() - cursor.execute("DROP TABLE IF EXISTS calculation_module") - cursor.execute("CREATE TABLE calculation_module (cm_id INTEGER NOT NULL, cm_name VARCHAR(255), wiki_url VARCHAR(255)," - "cm_description VARCHAR(255),cm_url VARCHAR(255),category VARCHAR(255),layers_needed VARCHAR(255),authorized_scale VARCHAR(255),description_link VARCHAR(255),createdAt REAL(255),updatedAt REAL(255),type_layer_needed REAL(255),vectors_needed REAL(255),type_vectors_needed REAL(255)," - " PRIMARY KEY(cm_id))") - conn.commit() - cursor.execute("DROP TABLE IF EXISTS inputs_calculation_module") - cursor.execute("CREATE TABLE inputs_calculation_module (input_id INTEGER NOT NULL, input_name VARCHAR(255), " - "input_type VARCHAR(255),input_parameter_name VARCHAR(255),input_value VARCHAR(255),input_priority INTEGER, input_unit VARCHAR(255)," - "input_min INTEGER,input_max INTEGER,createdAt REAL(255),updatedAt REAL(255),cm_id INTEGER NOT NULL," - " PRIMARY KEY(input_id),FOREIGN KEY(cm_id) REFERENCES calculation_module(cm_id))") - conn.commit() - return conn - -def register_calulation_module(data): - if data is not None: - conn = myCMpool.connect() - cursor = conn.cursor() - cm_name = data['cm_name'] - - - wiki_url = "" - try: - wiki_url = data['wiki_url'] - except: - pass - category = data['category'] - type_layer_needed = data['type_layer_needed'] - - cm_description = data['cm_description'] - cm_url = data['cm_url'] - cm_id = data['cm_id'] - layers_needed = data['layers_needed'] - authorized_scale = "[]" - try: - authorized_scale = data['authorized_scale'] - except: - pass - - description_link = "" - try: - description_link = data['description_link'] - except: - pass - vectors_needed = "[]" - try: - vectors_needed = data['vectors_needed'] - except: - pass - - type_vectors_needed = "[]" - try: - type_vectors_needed = data['type_vectors_needed'] - except: - pass - - updatedAt = datetime.utcnow() - createdAt = datetime.utcnow() - inputs_calculation_module = data['inputs_calculation_module'] - try: - - ln = str(layers_needed) - tn = str(type_layer_needed) - authorized_scale = str(authorized_scale) - description_link = str(description_link) - vectors_needed = str(vectors_needed) - type_vectors_needed = str(type_vectors_needed) - cursor.execute("INSERT INTO calculation_module (cm_id, cm_name, cm_description, category, cm_url, layers_needed, createdAt, updatedAt, type_layer_needed,authorized_scale,description_link,vectors_needed,type_vectors_needed,wiki_url) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", ( cm_id, cm_name, cm_description, category, cm_url, ln, createdAt, updatedAt,tn,authorized_scale,description_link,vectors_needed,type_vectors_needed,wiki_url )) - conn.commit() - for value in inputs_calculation_module: - input_name = value['input_name'] - input_type = value['input_type'] - input_parameter_name = value['input_parameter_name'] - input_value = str(value['input_value']) - input_priority = 0 - try: - input_priority = value['input_priority'] - except: - pass - input_unit = value['input_unit'] - input_min = value['input_min'] - input_max = value['input_max'] - cm_id = value['cm_id'] - conn = myCMpool.connect() - cursor = conn.cursor() - cursor.execute("INSERT INTO inputs_calculation_module (input_name, input_type, input_parameter_name, input_value,input_priority, input_unit, input_min, input_max, cm_id, createdAt,updatedAt) VALUES (?,?,?,?,?,?,?,?,?,?,?)", (input_name, input_type, input_parameter_name, input_value,input_priority, input_unit, input_min, input_max, cm_id, createdAt,updatedAt)) - - conn.commit() - conn.close() - - except ValidationError: - pass - except sqlite3.IntegrityError as e: - update_calulation_module(cm_id, cm_name, cm_description, category, cm_url, layers_needed, createdAt, updatedAt,type_layer_needed,authorized_scale,description_link,vectors_needed,inputs_calculation_module,cursor,conn,type_vectors_needed,wiki_url) - - -def update_calulation_module(cm_id, cm_name, cm_description, category, cm_url, layers_needed, createdAt, updatedAt, type_layer_needed,authorized_scale,description_link,vectors_needed ,inputs_calculation_module,cursor,conn,type_vectors_needed,wiki_url): - try: - ln = str(layers_needed) - tn = str(type_layer_needed) - auth_s = str(authorized_scale) - description_link = str(description_link) - vn = str(vectors_needed) - - type_vectors_needed = str(type_vectors_needed) - - cursor.execute("UPDATE calculation_module SET cm_name = ?, cm_description = ?, category= ?, cm_url= ?, layers_needed= ?, createdAt= ?, updatedAt = ? , type_layer_needed = ?, authorized_scale = ?,description_link = ?, vectors_needed = ?, type_vectors_needed=?, wiki_url=? WHERE cm_id = ? ", ( cm_name, cm_description, category, cm_url,ln , createdAt, updatedAt, tn,auth_s,description_link, vn,type_vectors_needed, wiki_url, cm_id )) - conn.commit() - cursor.execute("DELETE FROM inputs_calculation_module WHERE cm_id = ? ", (str(cm_id))) - conn.commit() - for value in inputs_calculation_module: - input_name = value['input_name'] - input_type = value['input_type'] - input_parameter_name = value['input_parameter_name'] - input_value = str(value['input_value']) - input_priority = 0 - try: - input_priority = value['input_priority'] - except: - pass - input_unit = value['input_unit'] - input_min = value['input_min'] - input_max = value['input_max'] - cm_id = value['cm_id'] - - cursor.execute("INSERT INTO inputs_calculation_module (input_name, input_type, input_parameter_name, input_value, input_priority, input_unit, input_min, input_max, cm_id, createdAt,updatedAt) VALUES (?,?,?,?,?,?,?,?,?,?,?)", (input_name, input_type, input_parameter_name, input_value,input_priority, input_unit, input_min, input_max, cm_id, createdAt,updatedAt)) - conn.commit() - conn.close() - - except ValidationError: - pass - except sqlite3.IntegrityError as e: - pass - -def getUI(cm_id): - conn = myCMpool.connect() - cursor = conn.cursor() - - - results = cursor.execute('select * from inputs_calculation_module where cm_id = ?', - (cm_id,)) - conn.commit() - response = helper.retrieve_list_from_sql_result(results) - - """ valid_condition = assert((cmd_id) in (get_all_cm_ids())() - if valid_condition: - results = cursor.execute('select * from inputs_calculation_module where cm_id = ?', - (cm_id)) - else: - return False()""" - conn.close() - return response - -def delete_cm(cm_id): - delete_cm_with_id(cm_id) - delete_cm_ui_with_id(cm_id) - -def delete_cm_ui_with_id(cm_id): - try: - conn = myCMpool.connect() - cursor = conn.cursor() - - results = cursor.execute('DELETE FROM inputs_calculation_module WHERE cm_id = ?', - (cm_id)) - conn.commit() - conn.close() - return results - - except ValidationError: - pass - except sqlite3.IntegrityError as e: - pass - -def delete_cm_with_id(cm_id): - try: - conn = myCMpool.connect() - cursor = conn.cursor() - - results = cursor.execute('DELETE FROM calculation_module WHERE cm_id = ?', - (cm_id)) - conn.commit() - conn.close() - return results - - except ValidationError: - pass - except sqlite3.IntegrityError as e: - pass - -def getCMList(): - response = helper.retrieve_list_from_sql_result(query_calculation_module_database('select * from calculation_module ')) - return response - -@celery.task(name = 'task-getConnection_db_gis') -def getConnection_db_gis(): - c = psycopg2.connect(get_connection_string()) - return c - -def get_connection_string(): - con = "host=" + HOST_DB + " user=" + USER_DB + " dbname=" + DATABASE_DB + " port=" + PORT_DB + " password=" + PASSWORD_DB + "" - return con - -def get_shapefile_from_selection(scalevalue, id_selected_list, ouput_directory, EPSG=str(3035)): - id_selected_list = helper.adapt_nuts_list(id_selected_list) - - output_shapefile = quote(helper.generate_shapefile_name(ouput_directory)) - if scalevalue == 'nuts': - subprocess.call('ogr2ogr -overwrite -f "ESRI Shapefile" '+output_shapefile+' PG:"'+get_connection_string()+'" -sql "select ST_Union(ST_Transform(geom,'+EPSG+')) from geo.nuts where nuts_id IN ('+ id_selected_list +') AND year = date({})"'.format("'2013-01-01'"), shell=True) - - else: - subprocess.call('ogr2ogr -overwrite -f "ESRI Shapefile" '+output_shapefile+' PG:"'+get_connection_string()+'" -sql "select ST_Union(ST_Transform(geom,'+EPSG+')) from public.tbl_lau1_2 where comm_id IN ('+ id_selected_list +')"', shell=True) - - return output_shapefile - -def get_raster_from_csv(wkt_point, layer_needed, output_directory): - inputs_raster_selection = {} - wkt_point_3035 = helper.projection_4326_to_3035(wkt_point) - filename_csv = helper.write_wkt_csv(helper.generate_csv_name(output_directory),wkt_point_3035) - for layer in layer_needed: - if 'layer_type' in layer: - type = layer['layer_type'] - id = layer['id'] - else: - type = layer['name'] - id = 0 - if id == 0: - dataset_directory = DATASET_DIRECTORY - directory = layer['workspaceName'] - root_path = dataset_directory + directory + "/data/" - path_to_dataset = root_path + layer['workspaceName'] + ".tif" - if not os.path.abspath(path_to_dataset).startswith(root_path): - raise Exception("directory traversal denied") - else: - upload = Uploads.query.get(layer['id']) - path_to_dataset = upload.url - - # create a file name as output - filename_tif = helper.generate_geotif_name(output_directory) - args = commands_in_array("gdalwarp -dstnodata 0 -cutline {} -crop_to_cutline -of GTiff {} {} -tr 100 100 -co COMPRESS=DEFLATE".format(filename_csv, path_to_dataset, filename_tif)) - run_command(args) - #os.system(com_string) - inputs_raster_selection[type] = filename_tif - return inputs_raster_selection - -def clip_raster_from_shapefile(shapefile_path,layer_needed, output_directory): - """ - - :param datasets_directory: input dataset directory - :param shapefile_path: input shapefile path - :param layer_needed: list of layer need for the CM - :param output_directory: output directory where we c - :return: dictionnary - """ - inputs_raster_selection = {} - # retrieve all layer neeeded - for layer in layer_needed: - if 'layer_type' in layer: - type = layer['layer_type'] - id = layer['id'] - else: - type = layer['name'] - id = 0 - if id == 0: - dataset_directory = DATASET_DIRECTORY - directory = layer['workspaceName'] - root_path = dataset_directory + directory + "/data/" - path_to_dataset = root_path + layer['workspaceName'] + ".tif" - if not os.path.abspath(path_to_dataset).startswith(root_path): - raise Exception("directory traversal denied") - else: - upload = Uploads.query.filter_by(id=layer['id']).first() - path_to_dataset = upload.url - # create a file name as output - filename_tif = helper.generate_geotif_name(output_directory) - # The previous option "-tr 100 100" seems to shift the layer - args = commands_in_array("gdalwarp -dstnodata 0 -cutline {} -crop_to_cutline -of GTiff {} {} -co COMPRESS=DEFLATE".format(shapefile_path, path_to_dataset, filename_tif)) - run_command(args) - inputs_raster_selection[type] = filename_tif - - - return inputs_raster_selection - - -def nuts2_within_the_selection_nuts_lau(scalevalue, nuts): - toCRS = 4258 - sql_query = sql_queries.nuts2_within_the_selection_nuts_lau(scalevalue, nuts, toCRS) - result = query_geographic_database(sql_query) - result = helper.retrieve_list_from_sql_result(result) - result = helper.from_dict_to_unique_array(result,'nuts_id') - return result - -def nuts_within_the_selection(geom): - toCRS = 4258 - sql_query = sql_queries.nuts_within_the_selection(geom,toCRS) - result = query_geographic_database(sql_query) - result = helper.retrieve_list_from_sql_result(result) - result = helper.from_dict_to_unique_array(result,'nuts_id') - return result - -def retrieve_vector_data_for_calculation_module(vectors_needed, scalevalue, area_selected): - """ - this function will return an array of vectors from the database - :param vectors_needed: - :param scalevalue: - :param area_selected: list of nut or geometry - :return: - """ - inputs_vectors_selection = {} - - for vector_table_requested in vectors_needed: - layer_path = '' - layer_id = vector_table_requested['id'] - layer_type = vector_table_requested['layer_type'] - if layer_id == 0: - if scalevalue == 'hectare': - layer_path = ExportCut.cut_hectares(area_selected, layer_type + "_ha", 'public', '2012') - print(layer_type + "_ha") - else: - layer_path = ExportCut.cut_nuts(layer_type + "_" + scalevalue, area_selected, 'public', '2012') - else: - - upload = Uploads.query.filter_by(id=layer_id).first() - path_to_dataset = upload.url - layer_path = ExportCut.cut_personal_layer(scalevalue, path_to_dataset, area_selected)['path'] - inputs_vectors_selection[layer_type] = layer_path - print(layer_path, layer_id, layer_type) - return inputs_vectors_selection - -def get_vectors_needed(cm_id): - conn = myCMpool.connect() - cursor = conn.cursor() - vectors_needed = cursor.execute('select vectors_needed from calculation_module where cm_id = ?', - (cm_id)) - conn.commit() - vectors_needed = vectors_needed.fetchone()[0] - vectors_needed = helper.unicode_array_to_string(vectors_needed) - conn.close() - return vectors_needed - - -def query_geographic_database(sql_query): - mypool = pool.QueuePool(getConnection_db_gis, max_overflow=100, pool_size=5) - # get a connectioncommands_in_array - conn = mypool.connect() - # use it - cursor = query(sql_query,conn) - return cursor - -def query_geographic_database_first(sql_query): - cursor = query_geographic_database(sql_query) - result = cursor.fetchone() - return result - -def check_table_existe(sql_query): - return query_geographic_database_first(sql_query) - -def query_calculation_module_database(sql_query): - - # get a connection - conn = myCMpool.connect() - # use it - cursor = conn.cursor() - - cursor.execute(sql_query) - conn.commit() - conn.close() - - - return cursor - -def query(sql_query,conn): - - # use it - cursor = conn.cursor() - - cursor.execute(sql_query) - conn.commit() - conn.close() - - - return cursor - -def get_cutline_input(areas, scalelevel, data_type): - if scalelevel == 'hectare': - areas = area_to_geom(areas) - if data_type == 'raster': - areas = projection_4326_to_3035(areas) - - return write_wkt_csv(generate_csv_name(constants.UPLOAD_DIRECTORY), areas) - else: - return get_shapefile_from_selection(scalelevel, areas, - constants.UPLOAD_DIRECTORY, '4326') - # if data_type == 'raster': - # if scalelevel == 'hectare': - # areas = area_to_geom(areas) - # cutline_input = write_wkt_csv(generate_csv_name(constants.UPLOAD_DIRECTORY), projection_4326_to_3035(areas)) # TODO: Projection to 3035 if raster - # else: - # cutline_input = get_shapefile_from_selection(scalelevel, areas, - # constants.UPLOAD_DIRECTORY, '4326') - # elif data_type == 'vector': - # if scalelevel == 'hectare': - # areas = area_to_geom(areas) - # cutline_input = write_wkt_csv(generate_csv_name(constants.UPLOAD_DIRECTORY), - # areas) - # else: - # cutline_input = get_shapefile_from_selection(scalelevel, areas, - # constants.UPLOAD_DIRECTORY, '4326') - # return cutline_input - -class ExportCut: - @staticmethod - def cut_nuts(layers: str, nuts: list, schema: str, year: str): - """ - The method called to cut a given list of nuts into a csv - :param layers: the layer selected - :param nuts: the list of nuts to export - :param schema: the DB schema - :param year: the data year - :return: - """ - csv_result = get_csv_from_nuts(layers=layers, nuts=nuts, schema=schema, year=year) - return ExportCut.save_file_csv_random_name(content=csv_result) - - @staticmethod - def cut_personal_layer(scale_level, upload_url, areas): - """ - The method called to cut a given list of nuts or a selection into a csv for a presonal layer - :param scale_level: nuts, lau or hectare - :param upload_url: the URL of the selected personal layer - :param areas: the selection on the map - :return: - """ - if scale_level == 'hectare': - areas = area_to_geom(areas) - cutline_input = write_wkt_csv(generate_csv_name(UPLOAD_DIRECTORY), areas) - else: - cutline_input = get_shapefile_from_selection(scale_level[:-1], areas, UPLOAD_DIRECTORY, '4326') - print(cutline_input) - cmd_cutline, output_csv = prepare_clip_personal_layer(cutline_input, upload_url) - args = app.helper.commands_in_array(cmd_cutline) - app.helper.run_command(args) - if not os.path.isfile(output_csv): - return { - "message": "not a csv file" - } - return { - "path": output_csv - } - - @staticmethod - def cut_hectares(areas: list, layers: str, schema: str, year: str): - """ - The method called to cut a given selection of hectares into a csv - :param areas: the area to cut - :param layers: the layer to select - :param schema: the DB schema - :param year: the data_year - :return: - """ - csv_result = get_csv_from_hectare(areas=areas, layers=layers, schema=schema, year=year) - return ExportCut.save_file_csv_random_name(content=csv_result) - - @staticmethod - def save_file_csv_random_name(content): - """ - Save a file into a temp folder with a random name - :param content: the content of the file you want to write - :return random_name: the name randomly generated - """ - path = ExportCut.generate_random_file_name() - content_str = content.getvalue() - with open(path, 'w', encoding='utf8') as f: - f.write(content_str) - return path - - @staticmethod - def generate_random_file_name(extension: str = '.csv'): - """ - generate a random file name - :param extension: the extension of the file, default to .csv - :return: the path of the generated file name or None if extension doesn't start with a dot - """ - # the extension must be an extension - if not extension.startswith('.'): - return None - - random_name = uuid.uuid4().hex + '.csv' - path = UPLOAD_DIRECTORY + '/' + random_name - return path - - -def get_csv_from_nuts(layers, nuts, schema, year): - """ - This method will generate a CSV from a nuts or list of nuts - :param layers: the layer selected - :param nuts: the selection - :param schema: the schema of the layer in the DB - :param year: the year of the data - :return:the csv containing the results - """ - # We must determine if it is a nuts or a lau - dateCol = "year" - schema2 = "geo" - if str(layers).endswith('lau2'): - layer_type = 'lau' - layer_name = layers[: -5] - id_type = 'comm_id' - layer_date = LAU_YEAR - dateCol = "date" - schema2 = "public" - - else: - scale = str(layers)[-5:] - layer_type = 'nuts' - layer_name = str(layers)[: -6] - id_type = 'nuts_id' - layer_date = NUTS_YEAR - if scale not in ['nuts3', 'nuts2', 'nuts1', 'nuts0']: - # allow co2 emission factors layer - if 'yearly_co2_emission_factors_view' in str(layers): - layer_name = 'yearly_co2_emission_factors_view' - else: - raise HugeRequestException(message=scale) - # handle special case of wwtp where geom column has a different name (manual integration) - geom_col_name = 'geometry' if layer_name.startswith('wwtp') else 'geom' - # check if year exists otherwise get most recent or fallback to default (1970) - # timestamp to year if necessary: SELECT TO_CHAR(timestamp :: DATE, 'yyyy') - date_sql = """SELECT timestamp FROM {0}.{1} GROUP BY timestamp ORDER BY timestamp DESC;""".format(schema, - layer_name) - try: - results = db.engine.execute(date_sql) - except: - raise RequestException("Failed retrieving year in database") - layer_year = year + '-01-01' - dates = [] - for row in results: - dates.append(row[0]) - if len(dates) == 0: - layer_year = '1970-01-01' - elif layer_year not in dates: - layer_year = dates[0] - # build query - sql = """ - WITH _ as (SELECT geom as _ from {6}.{3} WHERE {4} IN ({5}) AND {7} = '{8}-01-01') - SELECT ST_ASTEXT({9}) as geometry_wkt, ST_SRID({9}) as srid, {0}.{1}.* - FROM {0}.{1}, _ - WHERE timestamp = '{2}' - AND ST_Within({0}.{1}.{9}, st_transform( - _._, ST_SRID({9}) - )) - ;""".format( - schema, # 0 - layer_name, # 1 - layer_year, # 2 - layer_type, # 3 - id_type, # 4 - ', '.join("'{0}'".format(n) for n in nuts), # 5 - schema2, # 6 - dateCol, # 7 - layer_date, # 8 - geom_col_name # 9 - ) - # execute query - try: - result = db.engine.execute(sql) - except: - raise RequestException("Problem with your SQL query") - - # build CSV - return generate_csv_string(result) - - -def get_csv_from_hectare(areas, layers, schema, year): - """ - this method will generate a csv from hectare selection - :param areas: the selection - :param layers: the layer selected - :param schema: the schema of the layer in the db - :param year: the year of the data - :return: the csv containing the results - """ - if not str(layers).endswith('_ha'): - raise RequestException("this is not a correct layer for an hectare selection !") - # format the layer_name to contain only the name - layer_name = layers[:-3] - # build request - polyArray = [] - # convert to polygon format for each polygon and store them in polyArray - try: - for polygon in areas: - po = shapely_geom.Polygon([[p['lng'], p['lat']] for p in polygon['points']]) - polyArray.append(po) - except: - raise NotEnoughPointsException - # convert array of polygon into multipolygon - multipolygon = shapely_geom.MultiPolygon(polyArray) - # handle special case of wwtp where geom column has a different name (manual integration) - geom_col_name = 'geometry' if layer_name.startswith('wwtp') else 'geom' - # check if year exists otherwise get most recent or fallback to default (1970) - date_sql = """SELECT timestamp FROM {0}.{1} GROUP BY timestamp ORDER BY timestamp DESC;""".format(schema, - layer_name) - try: - results = db.engine.execute(date_sql) - except: - raise RequestException("Failed retrieving year in database") - layer_year = year + '-01-01' - dates = [] - for row in results: - dates.append(row[0]) - if len(dates) == 0: - layer_year = '1970-01-01' - elif layer_year not in dates: - layer_year = dates[0] - # build query - sql = """SELECT ST_ASTEXT({3}) as geometry_wkt, ST_SRID({3}) as srid, * - FROM {0}.{1} WHERE timestamp = '{2}' - AND ST_Within({0}.{1}.{3}, st_transform(st_geomfromtext('{4}', 4258), ST_SRID({3}) - ));""".format(schema, layer_name, layer_year, geom_col_name, str(multipolygon)) - # execute query - try: - result = db.engine.execute(sql) - except: - raise RequestException("Problem with your SQL query") - - - return generate_csv_string(result) - - -def prepare_clip_personal_layer(cutline_input, upload_url): - """ - Helper method to clip a personal layer - :param cutline_input: - :param upload_url: the url of the upload - :return: a tuple containing the command to use later ant the output csv path - """ - #upload_url += "data.csv" - output_csv = generate_csv_name(constants.UPLOAD_DIRECTORY) - cmd_cutline = "ogr2ogr -f 'CSV' -clipsrc {} {} {} -oo GEOM_POSSIBLE_NAMES=geometry_wkt -oo KEEP_GEOM_COLUMNS=NO".format( - cutline_input, output_csv, upload_url) - return cmd_cutline, output_csv +import os +import sqlite3 +import subprocess +import uuid +from datetime import datetime + +import app.helper +import psycopg2 +import shapely.geometry as shapely_geom +import sqlalchemy.pool as pool +from app import celery, constants +from app import dbGIS as db +from app import helper, sql_queries +from app.constants import ( + CM_DB_NAME, DATABASE_DB, DATASET_DIRECTORY, HOST_DB, LAU_YEAR, NUTS_YEAR, + PASSWORD_DB, PORT_DB, UPLOAD_DIRECTORY, USER_DB) +from app.decorators.exceptions import (HugeRequestException, + NotEnoughPointsException, + RequestException, ValidationError) + +from .helper import (area_to_geom, commands_in_array, generate_csv_name, + projection_4326_to_3035, run_command, write_wkt_csv) +from .models.uploads import Uploads, generate_csv_string + +try: + from shlex import quote +except ImportError: + from pipes import quote +try: + import ogr +except ImportError: + from osgeo import ogr + +try: + import osr +except ImportError: + from osgeo import osr +basedir = os.path.abspath(os.path.dirname(__file__)) + +db_path = os.path.join(basedir, '../data.sqlite') + +DB_NAME = CM_DB_NAME + +def getConnection_db_CM(): + c = sqlite3.connect(DB_NAME) + return c + + +myCMpool = pool.QueuePool(getConnection_db_CM, max_overflow=10, pool_size=15) + + +def addRegisterCalulationModule(data): + """ + this request will get the signature of a CM and will insert it to the database + :param data: signature payload of the CM + :return: + """ + + cm_name = data['cm_name'] + + + + wiki_url = "" + try: + wiki_url = data['wiki_url'] + except: + pass + category = data['category'] + type_layer_needed = data['type_layer_needed'] + authorized_scale = "[]" + try: + authorized_scale = data['authorized_scale'] + except: + pass + description_link = "" + try: + description_link = data['description_link'] + except: + pass + vectors_needed = "[]" + try: + vectors_needed = data['vectors_needed'] + except: + pass + + try: + type_vectors_needed = data['type_vectors_needed'] + except: + pass + cm_description = data['cm_description'] + cm_url = data['cm_url'] + cm_Id = data['id'] + layers_needed = data['layers_needed'] + updatedAt = datetime.utcnow() + createdAt = datetime.utcnow() + conn = myCMpool.connect() + cursor = conn.cursor() + cursor.execute("INSERT INTO calculation_module (cm_id, cm_name, cm_description, category, cm_url, layers_needed, createdAt, updateAt,type_layer_needed,authorized_scale,description_link,vectors_needed,wiki_url) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", ( cm_Id, cm_name, cm_description, category, cm_url, layers_needed, createdAt, updatedAt ,type_layer_needed,authorized_scale,description_link,vectors_needed,type_vectors_needed,wiki_url)) + cursor.close() + +def init_sqlite_caculation_module_database(dbname=DB_NAME): + """ + + :param dbname: this part will manually create the database for CM + :return: conn : this is the connection to the database + """ + conn = sqlite3.connect(dbname) + cursor = conn.cursor() + cursor.execute("DROP TABLE IF EXISTS calculation_module") + cursor.execute("CREATE TABLE calculation_module (cm_id INTEGER NOT NULL, cm_name VARCHAR(255), wiki_url VARCHAR(255)," + "cm_description VARCHAR(255),cm_url VARCHAR(255),category VARCHAR(255),layers_needed VARCHAR(255),authorized_scale VARCHAR(255),description_link VARCHAR(255),createdAt REAL(255),updatedAt REAL(255),type_layer_needed REAL(255),vectors_needed REAL(255),type_vectors_needed REAL(255)," + " PRIMARY KEY(cm_id))") + conn.commit() + cursor.execute("DROP TABLE IF EXISTS inputs_calculation_module") + cursor.execute("CREATE TABLE inputs_calculation_module (input_id INTEGER NOT NULL, input_name VARCHAR(255), " + "input_type VARCHAR(255),input_parameter_name VARCHAR(255),input_value VARCHAR(255),input_priority INTEGER, input_unit VARCHAR(255)," + "input_min INTEGER,input_max INTEGER,createdAt REAL(255),updatedAt REAL(255),cm_id INTEGER NOT NULL," + " PRIMARY KEY(input_id),FOREIGN KEY(cm_id) REFERENCES calculation_module(cm_id))") + conn.commit() + return conn + +def register_calulation_module(data): + if data is not None: + conn = myCMpool.connect() + cursor = conn.cursor() + cm_name = data['cm_name'] + + + wiki_url = "" + try: + wiki_url = data['wiki_url'] + except: + pass + category = data['category'] + type_layer_needed = data['type_layer_needed'] + + cm_description = data['cm_description'] + cm_url = data['cm_url'] + cm_id = data['cm_id'] + layers_needed = data['layers_needed'] + authorized_scale = "[]" + try: + authorized_scale = data['authorized_scale'] + except: + pass + + description_link = "" + try: + description_link = data['description_link'] + except: + pass + vectors_needed = "[]" + try: + vectors_needed = data['vectors_needed'] + except: + pass + + type_vectors_needed = "[]" + try: + type_vectors_needed = data['type_vectors_needed'] + except: + pass + + updatedAt = datetime.utcnow() + createdAt = datetime.utcnow() + inputs_calculation_module = data['inputs_calculation_module'] + try: + + ln = str(layers_needed) + tn = str(type_layer_needed) + authorized_scale = str(authorized_scale) + description_link = str(description_link) + vectors_needed = str(vectors_needed) + type_vectors_needed = str(type_vectors_needed) + cursor.execute("INSERT INTO calculation_module (cm_id, cm_name, cm_description, category, cm_url, layers_needed, createdAt, updatedAt, type_layer_needed,authorized_scale,description_link,vectors_needed,type_vectors_needed,wiki_url) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", ( cm_id, cm_name, cm_description, category, cm_url, ln, createdAt, updatedAt,tn,authorized_scale,description_link,vectors_needed,type_vectors_needed,wiki_url )) + conn.commit() + for value in inputs_calculation_module: + input_name = value['input_name'] + input_type = value['input_type'] + input_parameter_name = value['input_parameter_name'] + input_value = str(value['input_value']) + input_priority = 0 + try: + input_priority = value['input_priority'] + except: + pass + input_unit = value['input_unit'] + input_min = value['input_min'] + input_max = value['input_max'] + cm_id = value['cm_id'] + conn = myCMpool.connect() + cursor = conn.cursor() + cursor.execute("INSERT INTO inputs_calculation_module (input_name, input_type, input_parameter_name, input_value,input_priority, input_unit, input_min, input_max, cm_id, createdAt,updatedAt) VALUES (?,?,?,?,?,?,?,?,?,?,?)", (input_name, input_type, input_parameter_name, input_value,input_priority, input_unit, input_min, input_max, cm_id, createdAt,updatedAt)) + + conn.commit() + conn.close() + + except ValidationError: + pass + except sqlite3.IntegrityError as e: + update_calulation_module(cm_id, cm_name, cm_description, category, cm_url, layers_needed, createdAt, updatedAt,type_layer_needed,authorized_scale,description_link,vectors_needed,inputs_calculation_module,cursor,conn,type_vectors_needed,wiki_url) + + +def update_calulation_module(cm_id, cm_name, cm_description, category, cm_url, layers_needed, createdAt, updatedAt, type_layer_needed,authorized_scale,description_link,vectors_needed ,inputs_calculation_module,cursor,conn,type_vectors_needed,wiki_url): + try: + ln = str(layers_needed) + tn = str(type_layer_needed) + auth_s = str(authorized_scale) + description_link = str(description_link) + vn = str(vectors_needed) + + type_vectors_needed = str(type_vectors_needed) + + cursor.execute("UPDATE calculation_module SET cm_name = ?, cm_description = ?, category= ?, cm_url= ?, layers_needed= ?, createdAt= ?, updatedAt = ? , type_layer_needed = ?, authorized_scale = ?,description_link = ?, vectors_needed = ?, type_vectors_needed=?, wiki_url=? WHERE cm_id = ? ", ( cm_name, cm_description, category, cm_url,ln , createdAt, updatedAt, tn,auth_s,description_link, vn,type_vectors_needed, wiki_url, cm_id )) + conn.commit() + cursor.execute("DELETE FROM inputs_calculation_module WHERE cm_id = ? ", (str(cm_id))) + conn.commit() + for value in inputs_calculation_module: + input_name = value['input_name'] + input_type = value['input_type'] + input_parameter_name = value['input_parameter_name'] + input_value = str(value['input_value']) + input_priority = 0 + try: + input_priority = value['input_priority'] + except: + pass + input_unit = value['input_unit'] + input_min = value['input_min'] + input_max = value['input_max'] + cm_id = value['cm_id'] + + cursor.execute("INSERT INTO inputs_calculation_module (input_name, input_type, input_parameter_name, input_value, input_priority, input_unit, input_min, input_max, cm_id, createdAt,updatedAt) VALUES (?,?,?,?,?,?,?,?,?,?,?)", (input_name, input_type, input_parameter_name, input_value,input_priority, input_unit, input_min, input_max, cm_id, createdAt,updatedAt)) + conn.commit() + conn.close() + + except ValidationError: + pass + except sqlite3.IntegrityError as e: + pass + +def getUI(cm_id): + conn = myCMpool.connect() + cursor = conn.cursor() + + + results = cursor.execute('select * from inputs_calculation_module where cm_id = ?', + (cm_id,)) + conn.commit() + response = helper.retrieve_list_from_sql_result(results) + + """ valid_condition = assert((cmd_id) in (get_all_cm_ids())() + if valid_condition: + results = cursor.execute('select * from inputs_calculation_module where cm_id = ?', + (cm_id)) + else: + return False()""" + conn.close() + return response + +def delete_cm(cm_id): + delete_cm_with_id(cm_id) + delete_cm_ui_with_id(cm_id) + +def delete_cm_ui_with_id(cm_id): + try: + conn = myCMpool.connect() + cursor = conn.cursor() + + results = cursor.execute('DELETE FROM inputs_calculation_module WHERE cm_id = ?', + (cm_id)) + conn.commit() + conn.close() + return results + + except ValidationError: + pass + except sqlite3.IntegrityError as e: + pass + +def delete_cm_with_id(cm_id): + try: + conn = myCMpool.connect() + cursor = conn.cursor() + + results = cursor.execute('DELETE FROM calculation_module WHERE cm_id = ?', + (cm_id)) + conn.commit() + conn.close() + return results + + except ValidationError: + pass + except sqlite3.IntegrityError as e: + pass + +def getCMList(): + response = helper.retrieve_list_from_sql_result(query_calculation_module_database('select * from calculation_module ')) + return response + +@celery.task(name = 'task-getConnection_db_gis') +def getConnection_db_gis(): + c = psycopg2.connect(get_connection_string()) + return c + +def get_connection_string(): + con = "host=" + HOST_DB + " user=" + USER_DB + " dbname=" + DATABASE_DB + " port=" + PORT_DB + " password=" + PASSWORD_DB + "" + return con + +def get_shapefile_from_selection(scalevalue, id_selected_list, ouput_directory, EPSG=str(3035)): + id_selected_list = helper.adapt_nuts_list(id_selected_list) + + output_shapefile = quote(helper.generate_shapefile_name(ouput_directory)) + if scalevalue == 'nuts': + subprocess.call('ogr2ogr -overwrite -f "ESRI Shapefile" '+output_shapefile+' PG:"'+get_connection_string()+'" -sql "select ST_Union(ST_Transform(geom,'+EPSG+')) from geo.nuts where nuts_id IN ('+ id_selected_list +') AND year = date({})"'.format("'2013-01-01'"), shell=True) + + else: + subprocess.call('ogr2ogr -overwrite -f "ESRI Shapefile" '+output_shapefile+' PG:"'+get_connection_string()+'" -sql "select ST_Union(ST_Transform(geom,'+EPSG+')) from public.tbl_lau1_2 where comm_id IN ('+ id_selected_list +')"', shell=True) + + return output_shapefile + +def get_raster_from_csv(wkt_point, layer_needed, output_directory): + inputs_raster_selection = {} + wkt_point_3035 = helper.projection_4326_to_3035(wkt_point) + filename_csv = helper.write_wkt_csv(helper.generate_csv_name(output_directory),wkt_point_3035) + for layer in layer_needed: + if 'layer_type' in layer: + type = layer['layer_type'] + id = layer['id'] + else: + type = layer['name'] + id = 0 + if id == 0: + dataset_directory = DATASET_DIRECTORY + directory = layer['workspaceName'] + root_path = dataset_directory + directory + "/data/" + path_to_dataset = root_path + layer['workspaceName'] + ".tif" + if not os.path.abspath(path_to_dataset).startswith(root_path): + raise Exception("directory traversal denied") + else: + upload = Uploads.query.get(layer['id']) + path_to_dataset = upload.url + + # create a file name as output + filename_tif = helper.generate_geotif_name(output_directory) + args = commands_in_array("gdalwarp -dstnodata 0 -cutline {} -crop_to_cutline -of GTiff {} {} -tr 100 100 -co COMPRESS=DEFLATE".format(filename_csv, path_to_dataset, filename_tif)) + run_command(args) + #os.system(com_string) + inputs_raster_selection[type] = filename_tif + return inputs_raster_selection + +def clip_raster_from_shapefile(shapefile_path,layer_needed, output_directory): + """ + + :param datasets_directory: input dataset directory + :param shapefile_path: input shapefile path + :param layer_needed: list of layer need for the CM + :param output_directory: output directory where we c + :return: dictionnary + """ + inputs_raster_selection = {} + # retrieve all layer neeeded + for layer in layer_needed: + if 'layer_type' in layer: + type = layer['layer_type'] + id = layer['id'] + else: + type = layer['name'] + id = 0 + if id == 0: + dataset_directory = DATASET_DIRECTORY + directory = layer['workspaceName'] + root_path = dataset_directory + directory + "/data/" + path_to_dataset = root_path + layer['workspaceName'] + ".tif" + if not os.path.abspath(path_to_dataset).startswith(root_path): + raise Exception("directory traversal denied") + else: + upload = Uploads.query.filter_by(id=layer['id']).first() + path_to_dataset = upload.url + # create a file name as output + filename_tif = helper.generate_geotif_name(output_directory) + # The previous option "-tr 100 100" seems to shift the layer + args = commands_in_array("gdalwarp -dstnodata 0 -cutline {} -crop_to_cutline -of GTiff {} {} -co COMPRESS=DEFLATE".format(shapefile_path, path_to_dataset, filename_tif)) + run_command(args) + inputs_raster_selection[type] = filename_tif + + + return inputs_raster_selection + + +def nuts2_within_the_selection_nuts_lau(scalevalue, nuts): + toCRS = 4258 + sql_query = sql_queries.nuts2_within_the_selection_nuts_lau(scalevalue, nuts, toCRS) + result = query_geographic_database(sql_query) + result = helper.retrieve_list_from_sql_result(result) + result = helper.from_dict_to_unique_array(result,'nuts_id') + return result + +def nuts_within_the_selection(geom): + toCRS = 4258 + sql_query = sql_queries.nuts_within_the_selection(geom,toCRS) + result = query_geographic_database(sql_query) + result = helper.retrieve_list_from_sql_result(result) + result = helper.from_dict_to_unique_array(result,'nuts_id') + return result + +def retrieve_vector_data_for_calculation_module(vectors_needed, scalevalue, area_selected): + """ + this function will return an array of vectors from the database + :param vectors_needed: + :param scalevalue: + :param area_selected: list of nut or geometry + :return: + """ + inputs_vectors_selection = {} + + for vector_table_requested in vectors_needed: + layer_path = '' + layer_id = vector_table_requested['id'] + layer_type = vector_table_requested['layer_type'] + if layer_id == 0: + if scalevalue == 'hectare': + layer_path = ExportCut.cut_hectares(area_selected, layer_type + "_ha", 'public', '2012') + print(layer_type + "_ha") + else: + layer_path = ExportCut.cut_nuts(layer_type + "_" + scalevalue, area_selected, 'public', '2012') + else: + + upload = Uploads.query.filter_by(id=layer_id).first() + path_to_dataset = upload.url + layer_path = ExportCut.cut_personal_layer(scalevalue, path_to_dataset, area_selected)['path'] + inputs_vectors_selection[layer_type] = layer_path + print(layer_path, layer_id, layer_type) + return inputs_vectors_selection + +def get_vectors_needed(cm_id): + conn = myCMpool.connect() + cursor = conn.cursor() + vectors_needed = cursor.execute('select vectors_needed from calculation_module where cm_id = ?', + (cm_id)) + conn.commit() + vectors_needed = vectors_needed.fetchone()[0] + vectors_needed = helper.unicode_array_to_string(vectors_needed) + conn.close() + return vectors_needed + + +def query_geographic_database(sql_query): + mypool = pool.QueuePool(getConnection_db_gis, max_overflow=100, pool_size=5) + # get a connectioncommands_in_array + conn = mypool.connect() + # use it + cursor = query(sql_query,conn) + return cursor + +def query_geographic_database_first(sql_query): + cursor = query_geographic_database(sql_query) + result = cursor.fetchone() + return result + +def check_table_existe(sql_query): + return query_geographic_database_first(sql_query) + +def query_calculation_module_database(sql_query): + + # get a connection + conn = myCMpool.connect() + # use it + cursor = conn.cursor() + + cursor.execute(sql_query) + conn.commit() + conn.close() + + + return cursor + +def query(sql_query,conn): + + # use it + cursor = conn.cursor() + + cursor.execute(sql_query) + conn.commit() + conn.close() + + + return cursor + +def get_cutline_input(areas, scalelevel, data_type): + if scalelevel == 'hectare': + areas = area_to_geom(areas) + if data_type == 'raster': + areas = projection_4326_to_3035(areas) + + return write_wkt_csv(generate_csv_name(constants.UPLOAD_DIRECTORY), areas) + else: + return get_shapefile_from_selection(scalelevel, areas, + constants.UPLOAD_DIRECTORY, '4326') + # if data_type == 'raster': + # if scalelevel == 'hectare': + # areas = area_to_geom(areas) + # cutline_input = write_wkt_csv(generate_csv_name(constants.UPLOAD_DIRECTORY), projection_4326_to_3035(areas)) # TODO: Projection to 3035 if raster + # else: + # cutline_input = get_shapefile_from_selection(scalelevel, areas, + # constants.UPLOAD_DIRECTORY, '4326') + # elif data_type == 'vector': + # if scalelevel == 'hectare': + # areas = area_to_geom(areas) + # cutline_input = write_wkt_csv(generate_csv_name(constants.UPLOAD_DIRECTORY), + # areas) + # else: + # cutline_input = get_shapefile_from_selection(scalelevel, areas, + # constants.UPLOAD_DIRECTORY, '4326') + # return cutline_input + +class ExportCut: + @staticmethod + def cut_nuts(layers: str, nuts: list, schema: str, year: str): + """ + The method called to cut a given list of nuts into a csv + :param layers: the layer selected + :param nuts: the list of nuts to export + :param schema: the DB schema + :param year: the data year + :return: + """ + csv_result = get_csv_from_nuts(layers=layers, nuts=nuts, schema=schema, year=year) + return ExportCut.save_file_csv_random_name(content=csv_result) + + @staticmethod + def cut_personal_layer(scale_level, upload_url, areas): + """ + The method called to cut a given list of nuts or a selection into a csv for a presonal layer + :param scale_level: nuts, lau or hectare + :param upload_url: the URL of the selected personal layer + :param areas: the selection on the map + :return: + """ + if scale_level == 'hectare': + areas = area_to_geom(areas) + cutline_input = write_wkt_csv(generate_csv_name(UPLOAD_DIRECTORY), areas) + else: + cutline_input = get_shapefile_from_selection(scale_level[:-1], areas, UPLOAD_DIRECTORY, '4326') + print(cutline_input) + cmd_cutline, output_csv = prepare_clip_personal_layer(cutline_input, upload_url) + args = app.helper.commands_in_array(cmd_cutline) + app.helper.run_command(args) + if not os.path.isfile(output_csv): + return { + "message": "not a csv file" + } + return { + "path": output_csv + } + + @staticmethod + def cut_hectares(areas: list, layers: str, schema: str, year: str): + """ + The method called to cut a given selection of hectares into a csv + :param areas: the area to cut + :param layers: the layer to select + :param schema: the DB schema + :param year: the data_year + :return: + """ + csv_result = get_csv_from_hectare(areas=areas, layers=layers, schema=schema, year=year) + return ExportCut.save_file_csv_random_name(content=csv_result) + + @staticmethod + def save_file_csv_random_name(content): + """ + Save a file into a temp folder with a random name + :param content: the content of the file you want to write + :return random_name: the name randomly generated + """ + path = ExportCut.generate_random_file_name() + content_str = content.getvalue() + with open(path, 'w', encoding='utf8') as f: + f.write(content_str) + return path + + @staticmethod + def generate_random_file_name(extension: str = '.csv'): + """ + generate a random file name + :param extension: the extension of the file, default to .csv + :return: the path of the generated file name or None if extension doesn't start with a dot + """ + # the extension must be an extension + if not extension.startswith('.'): + return None + + random_name = uuid.uuid4().hex + '.csv' + path = UPLOAD_DIRECTORY + '/' + random_name + return path + + +def get_csv_from_nuts(layers, nuts, schema, year): + """ + This method will generate a CSV from a nuts or list of nuts + :param layers: the layer selected + :param nuts: the selection + :param schema: the schema of the layer in the DB + :param year: the year of the data + :return:the csv containing the results + """ + # We must determine if it is a nuts or a lau + dateCol = "year" + schema2 = "geo" + if str(layers).endswith('lau2'): + layer_type = 'lau' + layer_name = layers[: -5] + id_type = 'comm_id' + layer_date = LAU_YEAR + dateCol = "date" + schema2 = "public" + + else: + scale = str(layers)[-5:] + layer_type = 'nuts' + layer_name = str(layers)[: -6] + id_type = 'nuts_id' + layer_date = NUTS_YEAR + if scale not in ['nuts3', 'nuts2', 'nuts1', 'nuts0']: + # allow co2 emission factors layer + if 'yearly_co2_emission_factors_view' in str(layers): + layer_name = 'yearly_co2_emission_factors_view' + else: + raise HugeRequestException(message=scale) + # handle special case of wwtp where geom column has a different name (manual integration) + geom_col_name = 'geometry' if layer_name.startswith('wwtp') else 'geom' + # check if year exists otherwise get most recent or fallback to default (1970) + # timestamp to year if necessary: SELECT TO_CHAR(timestamp :: DATE, 'yyyy') + date_sql = """SELECT timestamp FROM {0}.{1} GROUP BY timestamp ORDER BY timestamp DESC;""".format(schema, + layer_name) + try: + results = db.engine.execute(date_sql) + except: + raise RequestException("Failed retrieving year in database") + layer_year = year + '-01-01' + dates = [] + for row in results: + dates.append(row[0]) + if len(dates) == 0: + layer_year = '1970-01-01' + elif layer_year not in dates: + layer_year = dates[0] + # build query + sql = """ + WITH _ as (SELECT geom as _ from {6}.{3} WHERE {4} IN ({5}) AND {7} = '{8}-01-01') + SELECT ST_ASTEXT({9}) as geometry_wkt, ST_SRID({9}) as srid, {0}.{1}.* + FROM {0}.{1}, _ + WHERE timestamp = '{2}' + AND ST_Within({0}.{1}.{9}, st_transform( + _._, ST_SRID({9}) + )) + ;""".format( + schema, # 0 + layer_name, # 1 + layer_year, # 2 + layer_type, # 3 + id_type, # 4 + ', '.join("'{0}'".format(n) for n in nuts), # 5 + schema2, # 6 + dateCol, # 7 + layer_date, # 8 + geom_col_name # 9 + ) + # execute query + try: + result = db.engine.execute(sql) + except: + raise RequestException("Problem with your SQL query") + + # build CSV + return generate_csv_string(result) + + +def get_csv_from_hectare(areas, layers, schema, year): + """ + this method will generate a csv from hectare selection + :param areas: the selection + :param layers: the layer selected + :param schema: the schema of the layer in the db + :param year: the year of the data + :return: the csv containing the results + """ + if not str(layers).endswith('_ha'): + raise RequestException("this is not a correct layer for an hectare selection !") + # format the layer_name to contain only the name + layer_name = layers[:-3] + # build request + polyArray = [] + # convert to polygon format for each polygon and store them in polyArray + try: + for polygon in areas: + po = shapely_geom.Polygon([[p['lng'], p['lat']] for p in polygon['points']]) + polyArray.append(po) + except: + raise NotEnoughPointsException + # convert array of polygon into multipolygon + multipolygon = shapely_geom.MultiPolygon(polyArray) + # handle special case of wwtp where geom column has a different name (manual integration) + geom_col_name = 'geometry' if layer_name.startswith('wwtp') else 'geom' + # check if year exists otherwise get most recent or fallback to default (1970) + date_sql = """SELECT timestamp FROM {0}.{1} GROUP BY timestamp ORDER BY timestamp DESC;""".format(schema, + layer_name) + try: + results = db.engine.execute(date_sql) + except: + raise RequestException("Failed retrieving year in database") + layer_year = year + '-01-01' + dates = [] + for row in results: + dates.append(row[0]) + if len(dates) == 0: + layer_year = '1970-01-01' + elif layer_year not in dates: + layer_year = dates[0] + # build query + sql = """SELECT ST_ASTEXT({3}) as geometry_wkt, ST_SRID({3}) as srid, * + FROM {0}.{1} WHERE timestamp = '{2}' + AND ST_Within({0}.{1}.{3}, st_transform(st_geomfromtext('{4}', 4258), ST_SRID({3}) + ));""".format(schema, layer_name, layer_year, geom_col_name, str(multipolygon)) + # execute query + try: + result = db.engine.execute(sql) + except: + raise RequestException("Problem with your SQL query") + + + return generate_csv_string(result) + + +def prepare_clip_personal_layer(cutline_input, upload_url): + """ + Helper method to clip a personal layer + :param cutline_input: + :param upload_url: the url of the upload + :return: a tuple containing the command to use later ant the output csv path + """ + #upload_url += "data.csv" + output_csv = generate_csv_name(constants.UPLOAD_DIRECTORY) + cmd_cutline = "ogr2ogr -f 'CSV' -clipsrc {} {} {} -oo GEOM_POSSIBLE_NAMES=geometry_wkt -oo KEEP_GEOM_COLUMNS=NO".format( + cutline_input, output_csv, upload_url) + return cmd_cutline, output_csv diff --git a/api/app/models/gdal2tiles.py b/api/app/models/gdal2tiles.py index fa59212a..9acb7599 100644 --- a/api/app/models/gdal2tiles.py +++ b/api/app/models/gdal2tiles.py @@ -38,6 +38,8 @@ # DEALINGS IN THE SOFTWARE. # ****************************************************************************** +import math +import os import sys try: @@ -48,8 +50,6 @@ print 'You are using "old gen" bindings. gdal2tiles needs "new gen" bindings.' sys.exit(1) -import os -import math try: from PIL import Image @@ -112,7 +112,6 @@ Class is available under the open-source GDAL license (www.gdal.org). """ -import math MAXZOOMLEVEL = 32 diff --git a/api/app/models/generalData.py b/api/app/models/generalData.py index 44ae90e9..4205f3d4 100644 --- a/api/app/models/generalData.py +++ b/api/app/models/generalData.py @@ -1,163 +1,164 @@ +from app.constants import (CRS_LAU, CRS_NUTS, CRS_USER_GEOMETRY, LAU_TABLE, + NUTS_LAU_LEVELS, NUTS_VAlUES) from app.models.indicators import * -from app.constants import CRS_USER_GEOMETRY, NUTS_VAlUES, NUTS_LAU_LEVELS, CRS_NUTS,CRS_LAU,LAU_TABLE def constructWithPartEachLayerHectare(geometry, year, layer, scale_level): - if len(layersData[layer]['indicators']) == 0 and scale_level not in layersData[layer]['data_lvl']: - return ' ' - query = '' - layer_table_name = layersData[layer]['schema_hectare']+"."+layersData[layer]['tablename'] - query_select = 'SELECT ' - from_part = 'stat_' + layer - scalelvl_column = '' - if 'scalelvl_column' in layersData[layer]: - scalelvl_column = layersData[layer]['scalelvl_column'] - if layersData[layer]['table_type'] == vector_type: - for indic in layersData[layer]['indicators']: - if 'table_column' in indic: - query_select+=get_indicator_as_query(indic,layer_table_name,layer,scalelvl_column, scale_level) - - - query_select = query_select[:-1] - query += from_part+" as (" - query += query_select - query += " FROM "+layer_table_name - year_query='' - if 'year' in layersData[layer]: - year_query="date = '{0}-01-01' and".format(layersData[layer]['year']) - if 'level_of_data' in layersData[layer] and NUTS_LAU_LEVELS[layersData[layer]['level_of_data']] < NUTS_LAU_LEVELS[scale_level]: - query += " WHERE "+ year_query+" ST_Intersects(st_transform(st_geomfromtext('"+ geometry +"'::text,"+CRS_USER_GEOMETRY+")," + layersData[layer]['crs'] + "),"+layer_table_name+"."+layersData[layer]['geo_column']+")" - else: - query += " WHERE "+ year_query+" ST_Within("+layer_table_name+"."+layersData[layer]['geo_column']+",st_transform(st_geomfromtext('"+ geometry +"'::text,"+CRS_USER_GEOMETRY+")," + layersData[layer]['crs'] + "))" - query += ")" - else: - agg_summary_stat = 'agg_summary_stat' - for indic in layersData[layer]['indicators']: - if 'table_column' in indic: - query_select += agg_summary_stat + '.'+indic['table_column']+' as '+ layer + indic['indicator_id'] + ',' - query_select = query_select[:-1] - query_select += " from (select (((ST_SummaryStatsAgg(ST_Clip("+ layersData[layer]['tablename'] + ".rast, 1, st_transform(st_geomfromtext('" + geometry + "'::text,"+CRS_USER_GEOMETRY+")," + layersData[layer]['crs'] + "),false),true,0))).*) as "+layersData[layer]['tablename'] - query += from_part+' AS ( ' - query += query_select - query += " FROM "+ layer_table_name - query += " WHERE ST_Intersects(st_transform(st_geomfromtext('"+ geometry +"'::text,"+CRS_USER_GEOMETRY+")," + layersData[layer]['crs'] + ")," + layersData[layer]['tablename'] + ".rast)) "+agg_summary_stat - query += ")" - - return query - -def constructWithPartEachLayerNutsLau(nuts, year, layer, scale_level): - # Get name of table to select nuts/lau - year='2013' - if len(layersData[layer]['indicators']) == 0 and scale_level not in layersData[layer]['data_lvl']: - return ' ' - query = '' - name_type = '' - scalelvl_column='' - nust_select_name = "nutsSelection_"+ layer - nuts_selection = None - if scale_level in NUTS_VAlUES: - scale_level_crs = CRS_NUTS - scale_level_name = 'nuts_id' - name_type = 'nuts' - fk_column_id = 'fk_nuts_gid' - nuts_selection = nust_select_name +" as (SELECT geom as geom from geo."+name_type+" where "+name_type+".year = date('"+year+"-01-01') and "+scale_level_name+" in ("+nuts+")), " - else: - scale_level_crs = CRS_LAU - scale_level_name = 'comm_id' - name_type = 'lau' - fk_column_id = 'fk_lau_gid' - nuts_selection = nust_select_name +" as (SELECT st_transform(geom,4326) as geom from public."+LAU_TABLE+" where "+scale_level_name+" in ("+nuts+")), " - if 'scalelvl_column' in layersData[layer]: - scalelvl_column = layersData[layer]['scalelvl_column'] - - - - - - #TODO: Make a nuts selection like heatload with within where clause in nuts selection. Do not use geom! Too slow - - - - - from_part = 'stat_' + layer - query_from_part = from_part+" as (" - layer_table_name = layersData[layer]['schema_scalelvl']+"."+layersData[layer]['tablename'] - query_select = 'SELECT ' - - - if layersData[layer]['table_type'] == 'raster' and layersData[layer]['data_aggregated']: #(layersData[layer]['tablename'] != 'wwtp' or layersData[layer]['tablename'] != 'industrial_database' or layersData[layer]['tablename'] != 'wind_50m'): - layer_table_name += '_'+name_type - - for indic in layersData[layer]['indicators']: - if 'table_column' in indic: - """ if 'diss_agg_method' in indic and indic['diss_agg_method'] == 'NUTS_result' and check_if_agg_or_dis_method(layer, scale_level): - get_dis_query(indic, layer_table_name, layer,scale_level_name) - else: """ - query_select+=get_indicator_as_query(indic, layer_table_name, layer,scalelvl_column,scale_level) - - query_select = query_select[:-1] - - - if layersData[layer]['data_aggregated'] == False: - query += nuts_selection - query += query_from_part - query += query_select - - - query_from =" from " + nust_select_name + ", "+layer_table_name - query += query_from - year_query='' - if 'year' in layersData[layer]: - year_query="date = '{0}-01-01' and".format(layersData[layer]['year']) - if check_if_agg_or_dis_method(layer, scale_level): - query += " where "+year_query+" st_within(st_centroid("+nust_select_name+".geom),st_transform("+layer_table_name+"."+layersData[layer]['geo_column']+","+scale_level_crs+"))) " - else: - - query += " where "+year_query+" st_within(st_transform("+layer_table_name+"."+layersData[layer]['geo_column']+","+scale_level_crs+"), "+nust_select_name+".geom)) " - else: - query += query_from_part - query += query_select - if scale_level in NUTS_VAlUES: - query += " FROM "+layer_table_name + ", geo." + name_type - query += " WHERE "+layer_table_name+"."+fk_column_id+" = geo."+name_type+".gid and "+name_type+".year = date('"+year+"-01-01') and "+layer_table_name+"."+scale_level_name+" IN ("+nuts+") ) " - else: - query += " FROM "+layer_table_name + ", public." + LAU_TABLE - query += " WHERE "+layer_table_name+"."+fk_column_id+" = public."+LAU_TABLE+".gid and "+layer_table_name+"."+scale_level_name+" IN ("+nuts+") ) " - - return query + if len(layersData[layer]['indicators']) == 0 and scale_level not in layersData[layer]['data_lvl']: + return ' ' + query = '' + layer_table_name = layersData[layer]['schema_hectare']+"."+layersData[layer]['tablename'] + query_select = 'SELECT ' + from_part = 'stat_' + layer + scalelvl_column = '' + if 'scalelvl_column' in layersData[layer]: + scalelvl_column = layersData[layer]['scalelvl_column'] + if layersData[layer]['table_type'] == vector_type: + for indic in layersData[layer]['indicators']: + if 'table_column' in indic: + query_select+=get_indicator_as_query(indic,layer_table_name,layer,scalelvl_column, scale_level) + + + query_select = query_select[:-1] + query += from_part+" as (" + query += query_select + query += " FROM "+layer_table_name + year_query='' + if 'year' in layersData[layer]: + year_query="date = '{0}-01-01' and".format(layersData[layer]['year']) + if 'level_of_data' in layersData[layer] and NUTS_LAU_LEVELS[layersData[layer]['level_of_data']] < NUTS_LAU_LEVELS[scale_level]: + query += " WHERE "+ year_query+" ST_Intersects(st_transform(st_geomfromtext('"+ geometry +"'::text,"+CRS_USER_GEOMETRY+")," + layersData[layer]['crs'] + "),"+layer_table_name+"."+layersData[layer]['geo_column']+")" + else: + query += " WHERE "+ year_query+" ST_Within("+layer_table_name+"."+layersData[layer]['geo_column']+",st_transform(st_geomfromtext('"+ geometry +"'::text,"+CRS_USER_GEOMETRY+")," + layersData[layer]['crs'] + "))" + query += ")" + else: + agg_summary_stat = 'agg_summary_stat' + for indic in layersData[layer]['indicators']: + if 'table_column' in indic: + query_select += agg_summary_stat + '.'+indic['table_column']+' as '+ layer + indic['indicator_id'] + ',' + query_select = query_select[:-1] + query_select += " from (select (((ST_SummaryStatsAgg(ST_Clip("+ layersData[layer]['tablename'] + ".rast, 1, st_transform(st_geomfromtext('" + geometry + "'::text,"+CRS_USER_GEOMETRY+")," + layersData[layer]['crs'] + "),false),true,0))).*) as "+layersData[layer]['tablename'] + query += from_part+' AS ( ' + query += query_select + query += " FROM "+ layer_table_name + query += " WHERE ST_Intersects(st_transform(st_geomfromtext('"+ geometry +"'::text,"+CRS_USER_GEOMETRY+")," + layersData[layer]['crs'] + ")," + layersData[layer]['tablename'] + ".rast)) "+agg_summary_stat + query += ")" + + return query + +def constructWithPartEachLayerNutsLau(nuts, year, layer, scale_level): + # Get name of table to select nuts/lau + year='2013' + if len(layersData[layer]['indicators']) == 0 and scale_level not in layersData[layer]['data_lvl']: + return ' ' + query = '' + name_type = '' + scalelvl_column='' + nust_select_name = "nutsSelection_"+ layer + nuts_selection = None + if scale_level in NUTS_VAlUES: + scale_level_crs = CRS_NUTS + scale_level_name = 'nuts_id' + name_type = 'nuts' + fk_column_id = 'fk_nuts_gid' + nuts_selection = nust_select_name +" as (SELECT geom as geom from geo."+name_type+" where "+name_type+".year = date('"+year+"-01-01') and "+scale_level_name+" in ("+nuts+")), " + else: + scale_level_crs = CRS_LAU + scale_level_name = 'comm_id' + name_type = 'lau' + fk_column_id = 'fk_lau_gid' + nuts_selection = nust_select_name +" as (SELECT st_transform(geom,4326) as geom from public."+LAU_TABLE+" where "+scale_level_name+" in ("+nuts+")), " + if 'scalelvl_column' in layersData[layer]: + scalelvl_column = layersData[layer]['scalelvl_column'] + + + + + + #TODO: Make a nuts selection like heatload with within where clause in nuts selection. Do not use geom! Too slow + + + + + from_part = 'stat_' + layer + query_from_part = from_part+" as (" + layer_table_name = layersData[layer]['schema_scalelvl']+"."+layersData[layer]['tablename'] + query_select = 'SELECT ' + + + if layersData[layer]['table_type'] == 'raster' and layersData[layer]['data_aggregated']: #(layersData[layer]['tablename'] != 'wwtp' or layersData[layer]['tablename'] != 'industrial_database' or layersData[layer]['tablename'] != 'wind_50m'): + layer_table_name += '_'+name_type + + for indic in layersData[layer]['indicators']: + if 'table_column' in indic: + """ if 'diss_agg_method' in indic and indic['diss_agg_method'] == 'NUTS_result' and check_if_agg_or_dis_method(layer, scale_level): + get_dis_query(indic, layer_table_name, layer,scale_level_name) + else: """ + query_select+=get_indicator_as_query(indic, layer_table_name, layer,scalelvl_column,scale_level) + + query_select = query_select[:-1] + + + if layersData[layer]['data_aggregated'] == False: + query += nuts_selection + query += query_from_part + query += query_select + + + query_from =" from " + nust_select_name + ", "+layer_table_name + query += query_from + year_query='' + if 'year' in layersData[layer]: + year_query="date = '{0}-01-01' and".format(layersData[layer]['year']) + if check_if_agg_or_dis_method(layer, scale_level): + query += " where "+year_query+" st_within(st_centroid("+nust_select_name+".geom),st_transform("+layer_table_name+"."+layersData[layer]['geo_column']+","+scale_level_crs+"))) " + else: + + query += " where "+year_query+" st_within(st_transform("+layer_table_name+"."+layersData[layer]['geo_column']+","+scale_level_crs+"), "+nust_select_name+".geom)) " + else: + query += query_from_part + query += query_select + if scale_level in NUTS_VAlUES: + query += " FROM "+layer_table_name + ", geo." + name_type + query += " WHERE "+layer_table_name+"."+fk_column_id+" = geo."+name_type+".gid and "+name_type+".year = date('"+year+"-01-01') and "+layer_table_name+"."+scale_level_name+" IN ("+nuts+") ) " + else: + query += " FROM "+layer_table_name + ", public." + LAU_TABLE + query += " WHERE "+layer_table_name+"."+fk_column_id+" = public."+LAU_TABLE+".gid and "+layer_table_name+"."+scale_level_name+" IN ("+nuts+") ) " + + return query def get_indicator_as_query(indic, layer_table_name, layer, scale_level_name, scale_level): - agg_method = indic['table_column'] - distinct='' - indic_id = 'as '+layer + indic['indicator_id'] + ',' - query_calcul = layer_table_name+'.' +indic['table_column'] + ')' + indic_id - if layer == 'agricultural_residues_view': - distinct = 'distinct ' - - switcher = { - 'sum':'sum('+query_calcul, - 'min':'min('+query_calcul, - 'max':'max('+query_calcul, - 'avg':'avg('+query_calcul, - 'mean_weighted_cell':'sum('+layer_table_name+'.count*'+layer_table_name+'.' +indic['table_column']+')/sum('+layer_table_name+'.count) '+indic_id, - 'mean_simple':'avg('+query_calcul, - 'NUTS_result':'sum('+layer_table_name+'.'+indic['table_column']+')/count('+distinct+layer_table_name+'.'+scale_level_name+') '+ indic_id - } - - if 'agg_method' in indic: - agg_method = indic['agg_method'] - if 'diss_agg_method' in indic and check_if_agg_or_dis_method(layer, scale_level): - agg_method = indic['diss_agg_method'] - if indic['table_column'] is 'mean': - quer = switcher.get(agg_method,switcher['mean_weighted_cell']) - else: - quer = switcher.get(agg_method,switcher['sum']) - - return quer - + agg_method = indic['table_column'] + distinct='' + indic_id = 'as '+layer + indic['indicator_id'] + ',' + query_calcul = layer_table_name+'.' +indic['table_column'] + ')' + indic_id + if layer == 'agricultural_residues_view': + distinct = 'distinct ' + + switcher = { + 'sum':'sum('+query_calcul, + 'min':'min('+query_calcul, + 'max':'max('+query_calcul, + 'avg':'avg('+query_calcul, + 'mean_weighted_cell':'sum('+layer_table_name+'.count*'+layer_table_name+'.' +indic['table_column']+')/sum('+layer_table_name+'.count) '+indic_id, + 'mean_simple':'avg('+query_calcul, + 'NUTS_result':'sum('+layer_table_name+'.'+indic['table_column']+')/count('+distinct+layer_table_name+'.'+scale_level_name+') '+ indic_id + } + + if 'agg_method' in indic: + agg_method = indic['agg_method'] + if 'diss_agg_method' in indic and check_if_agg_or_dis_method(layer, scale_level): + agg_method = indic['diss_agg_method'] + if indic['table_column'] is 'mean': + quer = switcher.get(agg_method,switcher['mean_weighted_cell']) + else: + quer = switcher.get(agg_method,switcher['sum']) + + return quer + def check_if_agg_or_dis_method(layer,scale_level): - return 'level_of_data' in layersData[layer] and NUTS_LAU_LEVELS[layersData[layer]['level_of_data']] < NUTS_LAU_LEVELS[scale_level] + return 'level_of_data' in layersData[layer] and NUTS_LAU_LEVELS[layersData[layer]['level_of_data']] < NUTS_LAU_LEVELS[scale_level] diff --git a/api/app/models/grids.py b/api/app/models/grids.py index c3a836ed..4503b8b9 100644 --- a/api/app/models/grids.py +++ b/api/app/models/grids.py @@ -1,4 +1,4 @@ -from app.models import dbGIS as db +from app.models import dbGIS as db from geoalchemy2 import Geometry @@ -21,4 +21,3 @@ class Grid1Km(db.Model): def __repr__(self): return "" % ( self.gid, self.id, self.xmin, self.xmax, self.ymin, self.ymax) - diff --git a/api/app/models/heat_density_map.py b/api/app/models/heat_density_map.py index dbfbbce2..96527b54 100644 --- a/api/app/models/heat_density_map.py +++ b/api/app/models/heat_density_map.py @@ -1,6 +1,8 @@ +from decimal import * + from app.models import db from geoalchemy2 import Raster -from decimal import * + #import logging #logging.basicConfig() #logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) diff --git a/api/app/models/heat_load_profile.py b/api/app/models/heat_load_profile.py index 34f8fa2b..4dfa327c 100644 --- a/api/app/models/heat_load_profile.py +++ b/api/app/models/heat_load_profile.py @@ -1,33 +1,32 @@ from app import dbGIS as db - #logging.basicConfig() #logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) class HeatLoadProfileNuts(db.Model): - __tablename__ = 'load_profile' - __table_args__ = ( - db.ForeignKeyConstraint(['fk_nuts_gid'], ['geo.nuts.gid'], name='load_profile_nuts_gid_fkey'), - db.ForeignKeyConstraint(['fk_time_id'], ['stat.time.id'], name='load_profile_time_id_fkey'), - {"schema": 'stat'} - ) - - CRS = 4258 - - id = db.Column(db.Integer, primary_key=True) - nuts_id = db.Column(db.String(14)) - process_id = db.Column(db.Integer) - process = db.Column(db.String()) - unit = db.Column(db.String()) - value = db.Column(db.Numeric(precision=30, scale=10)) - fk_nuts_gid = db.Column(db.BigInteger) - fk_time_id = db.Column(db.BigInteger) - - nuts = db.relationship("Nuts") - time = db.relationship("Time") - - def __repr__(self): - return "" % ( - self.nuts_id, str(self.time), self.value, self.unit) \ No newline at end of file + __tablename__ = 'load_profile' + __table_args__ = ( + db.ForeignKeyConstraint(['fk_nuts_gid'], ['geo.nuts.gid'], name='load_profile_nuts_gid_fkey'), + db.ForeignKeyConstraint(['fk_time_id'], ['stat.time.id'], name='load_profile_time_id_fkey'), + {"schema": 'stat'} + ) + + CRS = 4258 + + id = db.Column(db.Integer, primary_key=True) + nuts_id = db.Column(db.String(14)) + process_id = db.Column(db.Integer) + process = db.Column(db.String()) + unit = db.Column(db.String()) + value = db.Column(db.Numeric(precision=30, scale=10)) + fk_nuts_gid = db.Column(db.BigInteger) + fk_time_id = db.Column(db.BigInteger) + + nuts = db.relationship("Nuts") + time = db.relationship("Time") + + def __repr__(self): + return "" % ( + self.nuts_id, str(self.time), self.value, self.unit) diff --git a/api/app/models/heatloadQueries.py b/api/app/models/heatloadQueries.py index 2984c2ee..1a208fc8 100644 --- a/api/app/models/heatloadQueries.py +++ b/api/app/models/heatloadQueries.py @@ -1,363 +1,362 @@ -from app import constants -from app import model -from app import celery +from app import celery, constants, model + from .. import helper + #logging.basicConfig() #logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) class HeatLoadProfile: - @staticmethod - @celery.task(name = 'heatloadprofile_nuts_lau') - def heatloadprofile_nuts_lau(year, month, day, nuts, nuts_level): #/heat-load-profile/nuts-lau - request_type='' - # Check the type of the query - if month != 0 and day != 0: - request_type = 'day' - elif month != 0: - request_type = 'month' - else: - request_type = 'year' - - # Get the data - query = createQueryDataLPNutsLau(year=year, month=month, day=day, nuts=nuts,request_type=request_type, nuts_level=nuts_level) - - # Construction of the query - # Execution of the query - #query = db.session.execute(sql_query) - - res = model.query_geographic_database(query) - # Storing the results only if there is data - output = [] - - for c, q in enumerate(res): - if q[0]: - data={} - if request_type == 'year': - data = { - 'year': year,'month': q[4],'granularity': 'month','unit': 'kW','min': str(round(q[0], constants.NUMBER_DECIMAL_DATA)), - 'max': str(round(q[1], constants.NUMBER_DECIMAL_DATA)),'average': str(round(q[2], constants.NUMBER_DECIMAL_DATA)) - } - elif request_type == 'month': - data = { - 'year': year,'month': month,'day': q[4],'granularity': 'day','unit': 'kW','min': str(round(q[0], constants.NUMBER_DECIMAL_DATA)), - 'max': str(round(q[1], constants.NUMBER_DECIMAL_DATA)), - 'average': str(round(q[2], constants.NUMBER_DECIMAL_DATA)) - } - elif request_type == 'day': - data = { - 'year': year,'month': month,'day': day,'hour_of_day': q[4],'granularity': 'hour', - 'unit': 'kW','value': str(round(q[3], constants.NUMBER_DECIMAL_DATA)) - } - output.append(data) - else: - output = [] - - return { - "values": output - } - - - @staticmethod - @celery.task(name = 'heatloadprofile_hectares') - def heatloadprofile_hectares(year, month, day, geometry): #/heat-load-profile/hectares - - # Check the type of the query - if month != 0 and day != 0: - by = 'byDay' - elif month != 0: - by = 'byMonth' - else: - by = 'byYear' - - # Get the data - queryData = createQueryDataLPHectares(year=year, month=month, day=day, geometry=geometry) - - # Construction of the query - sql_query = queryData[by]['with'] + queryData[by]['select'] - # Execution of the query - #query = db.session.execute(sql_query) - query = model.query_geographic_database(sql_query) - # Storing the results only if there is data - output = [] - if by == 'byYear': - for c, q in enumerate(query): - if q[0]: - output.append({ - 'year': year, - 'month': q[3], - 'granularity': 'month', - 'unit': 'kW', - 'min': round(q[0], constants.NUMBER_DECIMAL_DATA), - 'max': round(q[1], constants.NUMBER_DECIMAL_DATA), - 'average': round(q[2], constants.NUMBER_DECIMAL_DATA) - }) - else: - output = [] - elif by == 'byMonth': - for c, q in enumerate(query): - if q[0]: - output.append({ - 'year': year, - 'month': month, - 'day': q[3], - 'granularity': 'day', - 'unit': 'kW', - 'min': round(q[0], constants.NUMBER_DECIMAL_DATA), - 'max': round(q[1], constants.NUMBER_DECIMAL_DATA), - 'average': round(q[2], constants.NUMBER_DECIMAL_DATA) - }) - else: - output = [] - else: - for c, q in enumerate(query): - if q[0]: - output.append({ - 'year': year, - 'month': month, - 'day': day, - 'hour_of_day': q[1], - 'granularity': 'hour', - 'unit': 'kW', - 'value': round(q[0], constants.NUMBER_DECIMAL_DATA) - }) - else: - output = [] - - return { - "values": output - } - - @staticmethod - def duration_curve_nuts_lau(year, nuts, nuts_level): #/heat-load-profile/duration-curve/nuts-lau - - # Get the query - sql_query = createQueryDataDCNutsLau(year=year, nuts=nuts, nuts_level=nuts_level) - - # Execution of the query - #query = db.session.execute(sql_query) - - query = model.query_geographic_database(sql_query) - - - # Store query results in a list - listAllValues = [] - points = [] - for n, q in enumerate(query): - #listAllValues.append(q[0]) - points.append({ - 'X':n+1, - 'Y':float(q[0]) - }) - - - newList = points[0:len(points)-1:10] - return newList - - @staticmethod - def duration_curve_hectares(year, geometry): #/heat-load-profile/duration-curve/hectares - - # Get the query - sql_query = createQueryDataDCHectares(year=year, geometry=geometry) - - query = model.query_geographic_database(sql_query) - listAllValues = [] - for n, q in enumerate(query): - listAllValues.append({ - 'X':n+1, - 'Y':q[0] - }) - - listAllValues = listAllValues[0:len(listAllValues)-1:10] - - - return listAllValues + @staticmethod + @celery.task(name = 'heatloadprofile_nuts_lau') + def heatloadprofile_nuts_lau(year, month, day, nuts, nuts_level): #/heat-load-profile/nuts-lau + request_type='' + # Check the type of the query + if month != 0 and day != 0: + request_type = 'day' + elif month != 0: + request_type = 'month' + else: + request_type = 'year' + + # Get the data + query = createQueryDataLPNutsLau(year=year, month=month, day=day, nuts=nuts,request_type=request_type, nuts_level=nuts_level) + + # Construction of the query + # Execution of the query + #query = db.session.execute(sql_query) + + res = model.query_geographic_database(query) + # Storing the results only if there is data + output = [] + + for c, q in enumerate(res): + if q[0]: + data={} + if request_type == 'year': + data = { + 'year': year,'month': q[4],'granularity': 'month','unit': 'kW','min': str(round(q[0], constants.NUMBER_DECIMAL_DATA)), + 'max': str(round(q[1], constants.NUMBER_DECIMAL_DATA)),'average': str(round(q[2], constants.NUMBER_DECIMAL_DATA)) + } + elif request_type == 'month': + data = { + 'year': year,'month': month,'day': q[4],'granularity': 'day','unit': 'kW','min': str(round(q[0], constants.NUMBER_DECIMAL_DATA)), + 'max': str(round(q[1], constants.NUMBER_DECIMAL_DATA)), + 'average': str(round(q[2], constants.NUMBER_DECIMAL_DATA)) + } + elif request_type == 'day': + data = { + 'year': year,'month': month,'day': day,'hour_of_day': q[4],'granularity': 'hour', + 'unit': 'kW','value': str(round(q[3], constants.NUMBER_DECIMAL_DATA)) + } + output.append(data) + else: + output = [] + + return { + "values": output + } + + + @staticmethod + @celery.task(name = 'heatloadprofile_hectares') + def heatloadprofile_hectares(year, month, day, geometry): #/heat-load-profile/hectares + + # Check the type of the query + if month != 0 and day != 0: + by = 'byDay' + elif month != 0: + by = 'byMonth' + else: + by = 'byYear' + + # Get the data + queryData = createQueryDataLPHectares(year=year, month=month, day=day, geometry=geometry) + + # Construction of the query + sql_query = queryData[by]['with'] + queryData[by]['select'] + # Execution of the query + #query = db.session.execute(sql_query) + query = model.query_geographic_database(sql_query) + # Storing the results only if there is data + output = [] + if by == 'byYear': + for c, q in enumerate(query): + if q[0]: + output.append({ + 'year': year, + 'month': q[3], + 'granularity': 'month', + 'unit': 'kW', + 'min': round(q[0], constants.NUMBER_DECIMAL_DATA), + 'max': round(q[1], constants.NUMBER_DECIMAL_DATA), + 'average': round(q[2], constants.NUMBER_DECIMAL_DATA) + }) + else: + output = [] + elif by == 'byMonth': + for c, q in enumerate(query): + if q[0]: + output.append({ + 'year': year, + 'month': month, + 'day': q[3], + 'granularity': 'day', + 'unit': 'kW', + 'min': round(q[0], constants.NUMBER_DECIMAL_DATA), + 'max': round(q[1], constants.NUMBER_DECIMAL_DATA), + 'average': round(q[2], constants.NUMBER_DECIMAL_DATA) + }) + else: + output = [] + else: + for c, q in enumerate(query): + if q[0]: + output.append({ + 'year': year, + 'month': month, + 'day': day, + 'hour_of_day': q[1], + 'granularity': 'hour', + 'unit': 'kW', + 'value': round(q[0], constants.NUMBER_DECIMAL_DATA) + }) + else: + output = [] + + return { + "values": output + } + + @staticmethod + def duration_curve_nuts_lau(year, nuts, nuts_level): #/heat-load-profile/duration-curve/nuts-lau + + # Get the query + sql_query = createQueryDataDCNutsLau(year=year, nuts=nuts, nuts_level=nuts_level) + + # Execution of the query + #query = db.session.execute(sql_query) + + query = model.query_geographic_database(sql_query) + + + # Store query results in a list + listAllValues = [] + points = [] + for n, q in enumerate(query): + #listAllValues.append(q[0]) + points.append({ + 'X':n+1, + 'Y':float(q[0]) + }) + + + newList = points[0:len(points)-1:10] + return newList + + @staticmethod + def duration_curve_hectares(year, geometry): #/heat-load-profile/duration-curve/hectares + + # Get the query + sql_query = createQueryDataDCHectares(year=year, geometry=geometry) + + query = model.query_geographic_database(sql_query) + listAllValues = [] + for n, q in enumerate(query): + listAllValues.append({ + 'X':n+1, + 'Y':q[0] + }) + + listAllValues = listAllValues[0:len(listAllValues)-1:10] + + + return listAllValues def createQueryDataLPHectares(year, month, day, geometry): - withPart = "with geomInput AS (SELECT ST_Transform(ST_GeomFromText('"+geometry+"',4326),4258) AS geometry), " + \ - "nuts2 AS (SELECT nuts.geom, nuts.gid FROM geo.nuts WHERE nuts.year = '2010-01-01' AND nuts.stat_levl_ = 2)," + \ - "subAreas as (SELECT ST_Transform(ST_Intersection(geomInput.geometry ,nuts2.geom),3035) as soustracteGeom, " +\ - "nuts2.gid " +\ - "FROM nuts2, geomInput " +\ - "where ST_Intersects(nuts2.geom,geomInput.geometry)), " + \ - "statBySubAreas as (SELECT (ST_SummaryStatsAgg(ST_Clip(heat_tot_curr_density.rast,1, subAreas.soustracteGeom,0,true),1,true)).* as stat, subAreas.gid " +\ - "FROM subAreas, geo.heat_tot_curr_density, geomInput " +\ - "WHERE ST_Intersects(heat_tot_curr_density.rast,subAreas.soustracteGeom) group by subAreas.gid), " +\ - "statLoadProfilBySubarea as (select stat.load_profile.nuts_id as load_profile_nutsid, stat.load_profile.value as val_load_profile, " +\ - "stat.time.month as month_of_year, " +\ - "stat.time.hour_of_year as hour_of_year, " +\ - "stat.time.day as day_of_month, " +\ - "stat.time.hour_of_day as hour_of_day, " +\ - "statBySubAreas.count as statCount, statBySubAreas.sum as statSum_HD " +\ - "from stat.load_profile " +\ - "left join statBySubAreas on statBySubAreas.gid = stat.load_profile.fk_nuts_gid " +\ - "inner join stat.time on stat.load_profile.fk_time_id = stat.time.id " +\ - "WHERE fk_nuts_gid is not null and fk_time_id is not null " +\ - "AND statBySubAreas.gid = stat.load_profile.fk_nuts_gid AND stat.time.year = " + str(year) + " " +\ - "order by stat.time.hour_of_year), " +\ - "totalLoadprofile as ( " +\ - "select sum(val_load_profile) as tot_load_profile,load_profile_nutsid " +\ - "from statLoadProfilBySubarea group by load_profile_nutsid), " +\ - "normalizedData as (select sum(val_load_profile/tot_load_profile*statSum_HD) as normalizedCalutation, " +\ - "hour_of_year, month_of_year, day_of_month, hour_of_day " +\ - "from statLoadProfilBySubarea " +\ - "inner join totalLoadprofile on statLoadProfilBySubarea.load_profile_nutsid = totalLoadprofile.load_profile_nutsid " +\ - "group by hour_of_year, month_of_year, hour_of_day, day_of_month " +\ - "order by hour_of_year) " +\ - "select " - - selectYear = "min(normalizedCalutation), max(normalizedCalutation), avg(normalizedCalutation), month_of_year " +\ - "from normalizedData " +\ - "group by month_of_year " +\ - "order by month_of_year" - selectMonth = "min(normalizedCalutation), max(normalizedCalutation), avg(normalizedCalutation), day_of_month " +\ - "from normalizedData " +\ - "where month_of_year = " + str(month) + " " +\ - "group by day_of_month, month_of_year " +\ - "order by day_of_month" - selectDay = "normalizedCalutation, hour_of_day " +\ - "from normalizedData " +\ - "where day_of_month = " + str(day) + " " +\ - "and month_of_year = " + str(month) + " " +\ - "group by normalizedCalutation, hour_of_day " +\ - "order by hour_of_day" - - # Dictionary with query data - queryData = {'byYear':{'with':withPart, 'select':selectYear}, - 'byMonth':{'with':withPart, 'select':selectMonth}, - 'byDay':{'with':withPart, 'select':selectDay}} - - return queryData + withPart = "with geomInput AS (SELECT ST_Transform(ST_GeomFromText('"+geometry+"',4326),4258) AS geometry), " + \ + "nuts2 AS (SELECT nuts.geom, nuts.gid FROM geo.nuts WHERE nuts.year = '2010-01-01' AND nuts.stat_levl_ = 2)," + \ + "subAreas as (SELECT ST_Transform(ST_Intersection(geomInput.geometry ,nuts2.geom),3035) as soustracteGeom, " +\ + "nuts2.gid " +\ + "FROM nuts2, geomInput " +\ + "where ST_Intersects(nuts2.geom,geomInput.geometry)), " + \ + "statBySubAreas as (SELECT (ST_SummaryStatsAgg(ST_Clip(heat_tot_curr_density.rast,1, subAreas.soustracteGeom,0,true),1,true)).* as stat, subAreas.gid " +\ + "FROM subAreas, geo.heat_tot_curr_density, geomInput " +\ + "WHERE ST_Intersects(heat_tot_curr_density.rast,subAreas.soustracteGeom) group by subAreas.gid), " +\ + "statLoadProfilBySubarea as (select stat.load_profile.nuts_id as load_profile_nutsid, stat.load_profile.value as val_load_profile, " +\ + "stat.time.month as month_of_year, " +\ + "stat.time.hour_of_year as hour_of_year, " +\ + "stat.time.day as day_of_month, " +\ + "stat.time.hour_of_day as hour_of_day, " +\ + "statBySubAreas.count as statCount, statBySubAreas.sum as statSum_HD " +\ + "from stat.load_profile " +\ + "left join statBySubAreas on statBySubAreas.gid = stat.load_profile.fk_nuts_gid " +\ + "inner join stat.time on stat.load_profile.fk_time_id = stat.time.id " +\ + "WHERE fk_nuts_gid is not null and fk_time_id is not null " +\ + "AND statBySubAreas.gid = stat.load_profile.fk_nuts_gid AND stat.time.year = " + str(year) + " " +\ + "order by stat.time.hour_of_year), " +\ + "totalLoadprofile as ( " +\ + "select sum(val_load_profile) as tot_load_profile,load_profile_nutsid " +\ + "from statLoadProfilBySubarea group by load_profile_nutsid), " +\ + "normalizedData as (select sum(val_load_profile/tot_load_profile*statSum_HD) as normalizedCalutation, " +\ + "hour_of_year, month_of_year, day_of_month, hour_of_day " +\ + "from statLoadProfilBySubarea " +\ + "inner join totalLoadprofile on statLoadProfilBySubarea.load_profile_nutsid = totalLoadprofile.load_profile_nutsid " +\ + "group by hour_of_year, month_of_year, hour_of_day, day_of_month " +\ + "order by hour_of_year) " +\ + "select " + + selectYear = "min(normalizedCalutation), max(normalizedCalutation), avg(normalizedCalutation), month_of_year " +\ + "from normalizedData " +\ + "group by month_of_year " +\ + "order by month_of_year" + selectMonth = "min(normalizedCalutation), max(normalizedCalutation), avg(normalizedCalutation), day_of_month " +\ + "from normalizedData " +\ + "where month_of_year = " + str(month) + " " +\ + "group by day_of_month, month_of_year " +\ + "order by day_of_month" + selectDay = "normalizedCalutation, hour_of_day " +\ + "from normalizedData " +\ + "where day_of_month = " + str(day) + " " +\ + "and month_of_year = " + str(month) + " " +\ + "group by normalizedCalutation, hour_of_day " +\ + "order by hour_of_day" + + # Dictionary with query data + queryData = {'byYear':{'with':withPart, 'select':selectYear}, + 'byMonth':{'with':withPart, 'select':selectMonth}, + 'byDay':{'with':withPart, 'select':selectDay}} + + return queryData # ALL QUERIES DATA FOR THE HEAT LOAD PROFILE BY NUTS def createQueryDataLPNutsLau(year, month, day, nuts, request_type, nuts_level, query_type="heatload"): - where_request='' - nutsSelectionQuery='' - scale_schema = 'geo' - hd_nuts_select= '' - from_clause_lp = 'stat.load_profile' - - if request_type=='year': - time_columns = "stat.time.month AS statmonth,stat.time.year AS statyear" - group_by_time_columns="statmonth,statyear" - elif request_type == 'month': - time_columns = "stat.time.day AS statday,stat.time.month AS statmonth,stat.time.year AS statyear" - group_by_time_columns=" statday, statmonth, statyear" - where_request="where statmonth = " + str(month) - elif request_type == 'day': - time_columns = "stat.time.hour_of_day as hour_of_day, stat.time.day AS statday,stat.time.month AS statmonth,stat.time.year AS statyear" - group_by_time_columns="hour_of_day, statday, statmonth, statyear" - where_request="where statmonth = " + str(month) + " and statday = " + str(day) - if nuts_level == 'LAU 2': - scale_level_table='lau' - scale_id = 'comm_id' - else: - scale_level_table='nuts' - scale_id = 'nuts_id' - - hd_table = "stat.heat_tot_curr_density_"+scale_level_table - where_clause_hd = hd_table+"."+scale_id+" in ("+nuts+")" - - from_hd = hd_table - - if nuts_level in constants.scale_level_loadprofile_aggreagtion: - nutsSelectionQuery = helper.get_nuts_query_selection(nuts,scale_level_table, scale_id) - - where_clause_lp = "stat.load_profile.nuts_id = nutsSelection.nuts2_id" - from_clause_lp += ',nutsSelection' - hd_nuts_select = "nutsSelection.nuts2_id" - from_hd += ', nutsSelection' - where_clause_hd = hd_table+"."+scale_id+" = nutsSelection.scale_id" - - elif nuts_level == 'NUTS 2': - where_clause_lp = "stat.load_profile.nuts_id in ("+nuts+")" - where_clause_hd = hd_table+"."+scale_id+" in ("+nuts+")" - hd_nuts_select = hd_table + '.nuts_id' - - query_lp = """loadprofile as ( - SELECT sum(stat.load_profile.value) as valtot, stat.load_profile.nuts_id - from """+from_clause_lp+""" - where """+where_clause_lp+""" - group by stat.load_profile.nuts_id - ), """ - - query_hd = """heatdemand as ( - SELECT sum(sum) as HDtotal,"""+hd_nuts_select+""" as nuts2_id - from """+from_hd+""" - where """+where_clause_hd+""" - group by """+hd_nuts_select+""" - ), """ - - select_normalized = '' - groupby_normalized = '' - query_select='' - if query_type == 'duration_curve': - select_normalized = 'sum(stat.load_profile.value/valtot*HDtotal) as val, stat.time.hour_of_year as hoy' - groupby_normalized = "stat.load_profile.fk_nuts_gid,stat.time.hour_of_year HAVING COUNT(value) = COUNT(*)" - query_select = "select sum(val) as values, hoy from normalizedData group by hoy order by values DESC" - - - elif query_type == 'heatload': - select_normalized = "avg(stat.load_profile.value / valtot * HDtotal) AS avg_1,min(stat.load_profile.value / valtot * HDtotal) AS min_1, max(stat.load_profile.value / valtot * HDtotal) AS max_1, sum(stat.load_profile.value / valtot * HDtotal) as sum_1,"+time_columns - groupby_normalized = " stat.load_profile.fk_nuts_gid, """+group_by_time_columns - query_select="""select sum(min_1), sum(max_1), sum(avg_1), sum(sum_1), """+group_by_time_columns - query_select+=""" from normalizedData """ - query_select+=where_request+""" group by """+group_by_time_columns+""" order by """+group_by_time_columns - - - - query_normalized = """normalizedData as ( - SELECT """+select_normalized+""" - FROM """+from_clause_lp+""", heatdemand hd, loadprofile lp, stat.time - where """+where_clause_lp+""" - and stat.load_profile.nuts_id is not null and - stat.load_profile.fk_time_id is not null and - stat.time.id = stat.load_profile.fk_time_id and - stat.load_profile.nuts_id = hd.nuts2_id and - stat.load_profile.nuts_id = lp.nuts_id - group by """+groupby_normalized+""")""" - - - query = 'with ' + nutsSelectionQuery + query_lp + query_hd + query_normalized + query_select - return query + where_request='' + nutsSelectionQuery='' + scale_schema = 'geo' + hd_nuts_select= '' + from_clause_lp = 'stat.load_profile' + + if request_type=='year': + time_columns = "stat.time.month AS statmonth,stat.time.year AS statyear" + group_by_time_columns="statmonth,statyear" + elif request_type == 'month': + time_columns = "stat.time.day AS statday,stat.time.month AS statmonth,stat.time.year AS statyear" + group_by_time_columns=" statday, statmonth, statyear" + where_request="where statmonth = " + str(month) + elif request_type == 'day': + time_columns = "stat.time.hour_of_day as hour_of_day, stat.time.day AS statday,stat.time.month AS statmonth,stat.time.year AS statyear" + group_by_time_columns="hour_of_day, statday, statmonth, statyear" + where_request="where statmonth = " + str(month) + " and statday = " + str(day) + if nuts_level == 'LAU 2': + scale_level_table='lau' + scale_id = 'comm_id' + else: + scale_level_table='nuts' + scale_id = 'nuts_id' + + hd_table = "stat.heat_tot_curr_density_"+scale_level_table + where_clause_hd = hd_table+"."+scale_id+" in ("+nuts+")" + + from_hd = hd_table + + if nuts_level in constants.scale_level_loadprofile_aggreagtion: + nutsSelectionQuery = helper.get_nuts_query_selection(nuts,scale_level_table, scale_id) + + where_clause_lp = "stat.load_profile.nuts_id = nutsSelection.nuts2_id" + from_clause_lp += ',nutsSelection' + hd_nuts_select = "nutsSelection.nuts2_id" + from_hd += ', nutsSelection' + where_clause_hd = hd_table+"."+scale_id+" = nutsSelection.scale_id" + + elif nuts_level == 'NUTS 2': + where_clause_lp = "stat.load_profile.nuts_id in ("+nuts+")" + where_clause_hd = hd_table+"."+scale_id+" in ("+nuts+")" + hd_nuts_select = hd_table + '.nuts_id' + + query_lp = """loadprofile as ( + SELECT sum(stat.load_profile.value) as valtot, stat.load_profile.nuts_id + from """+from_clause_lp+""" + where """+where_clause_lp+""" + group by stat.load_profile.nuts_id + ), """ + + query_hd = """heatdemand as ( + SELECT sum(sum) as HDtotal,"""+hd_nuts_select+""" as nuts2_id + from """+from_hd+""" + where """+where_clause_hd+""" + group by """+hd_nuts_select+""" + ), """ + + select_normalized = '' + groupby_normalized = '' + query_select='' + if query_type == 'duration_curve': + select_normalized = 'sum(stat.load_profile.value/valtot*HDtotal) as val, stat.time.hour_of_year as hoy' + groupby_normalized = "stat.load_profile.fk_nuts_gid,stat.time.hour_of_year HAVING COUNT(value) = COUNT(*)" + query_select = "select sum(val) as values, hoy from normalizedData group by hoy order by values DESC" + + + elif query_type == 'heatload': + select_normalized = "avg(stat.load_profile.value / valtot * HDtotal) AS avg_1,min(stat.load_profile.value / valtot * HDtotal) AS min_1, max(stat.load_profile.value / valtot * HDtotal) AS max_1, sum(stat.load_profile.value / valtot * HDtotal) as sum_1,"+time_columns + groupby_normalized = " stat.load_profile.fk_nuts_gid, """+group_by_time_columns + query_select="""select sum(min_1), sum(max_1), sum(avg_1), sum(sum_1), """+group_by_time_columns + query_select+=""" from normalizedData """ + query_select+=where_request+""" group by """+group_by_time_columns+""" order by """+group_by_time_columns + + + + query_normalized = """normalizedData as ( + SELECT """+select_normalized+""" + FROM """+from_clause_lp+""", heatdemand hd, loadprofile lp, stat.time + where """+where_clause_lp+""" + and stat.load_profile.nuts_id is not null and + stat.load_profile.fk_time_id is not null and + stat.time.id = stat.load_profile.fk_time_id and + stat.load_profile.nuts_id = hd.nuts2_id and + stat.load_profile.nuts_id = lp.nuts_id + group by """+groupby_normalized+""")""" + + + query = 'with ' + nutsSelectionQuery + query_lp + query_hd + query_normalized + query_select + return query # ALL QUERIES DATA FOR THE DURATION CURVE BY NUTS def createQueryDataDCNutsLau(year, nuts, nuts_level): - sql_query = createQueryDataLPNutsLau(year,None,None,nuts,'year',nuts_level,'duration_curve') - return sql_query + sql_query = createQueryDataLPNutsLau(year,None,None,nuts,'year',nuts_level,'duration_curve') + return sql_query # ALL QUERIES DATA FOR THE DURATION CURVE BY HECTARES def createQueryDataDCHectares(year, geometry): - sql_query = "with geomInput AS (SELECT ST_Transform(ST_GeomFromText('"+geometry+"',4326),4258) AS geometry), " + \ - "nuts2 AS (SELECT nuts.geom, nuts.gid FROM geo.nuts WHERE nuts.year = '2010-01-01' AND nuts.stat_levl_ = 2)," + \ - "subAreas as (SELECT ST_Transform(ST_Intersection(geomInput.geometry ,nuts2.geom),3035) as soustracteGeom, " + \ - "nuts2.gid " + \ - "FROM nuts2, geomInput " + \ - "where ST_Intersects(nuts2.geom,geomInput.geometry)), " + \ - "statBySubAreas as (SELECT (ST_SummaryStatsAgg(ST_Clip(heat_tot_curr_density.rast,1, " +\ - "subAreas.soustracteGeom,0,true),1,true)).* as stat, subAreas.gid " +\ - "FROM subAreas, geo.heat_tot_curr_density " +\ - "WHERE ST_Intersects(heat_tot_curr_density.rast,subAreas.soustracteGeom) group by subAreas.gid), " +\ - "statLoadProfilBySubarea as (select stat.load_profile.nuts_id as load_profile_nutsid, stat.load_profile.value as val_load_profile, " +\ - "stat.time.month as month_of_year, " +\ - "stat.time.hour_of_year as hour_of_year, " +\ - "stat.time.day as day_of_month, " +\ - "stat.time.hour_of_day as hour_of_day, " +\ - "statBySubAreas.count as statCount, statBySubAreas.sum as statSum_HD " +\ - "from stat.load_profile " +\ - "left join statBySubAreas on statBySubAreas.gid = stat.load_profile.fk_nuts_gid " +\ - "inner join stat.time on stat.load_profile.fk_time_id = stat.time.id " +\ - "WHERE fk_nuts_gid is not null and fk_time_id is not null " +\ - "AND statBySubAreas.gid = stat.load_profile.fk_nuts_gid AND stat.time.year = " + str(year) + " " +\ - "order by stat.time.hour_of_year), " +\ - "totalLoadprofile as ( " +\ - "select sum(val_load_profile) as tot_load_profile,load_profile_nutsid " +\ - "from statLoadProfilBySubarea group by load_profile_nutsid) " +\ - "select sum(val_load_profile/tot_load_profile*statSum_HD) as normalizedCalutation,hour_of_year " +\ - "from statLoadProfilBySubarea " +\ - "inner join totalLoadprofile on statLoadProfilBySubarea.load_profile_nutsid = totalLoadprofile.load_profile_nutsid " +\ - "group by hour_of_year " +\ - "order by normalizedCalutation DESC;" - - return sql_query - + sql_query = "with geomInput AS (SELECT ST_Transform(ST_GeomFromText('"+geometry+"',4326),4258) AS geometry), " + \ + "nuts2 AS (SELECT nuts.geom, nuts.gid FROM geo.nuts WHERE nuts.year = '2010-01-01' AND nuts.stat_levl_ = 2)," + \ + "subAreas as (SELECT ST_Transform(ST_Intersection(geomInput.geometry ,nuts2.geom),3035) as soustracteGeom, " + \ + "nuts2.gid " + \ + "FROM nuts2, geomInput " + \ + "where ST_Intersects(nuts2.geom,geomInput.geometry)), " + \ + "statBySubAreas as (SELECT (ST_SummaryStatsAgg(ST_Clip(heat_tot_curr_density.rast,1, " +\ + "subAreas.soustracteGeom,0,true),1,true)).* as stat, subAreas.gid " +\ + "FROM subAreas, geo.heat_tot_curr_density " +\ + "WHERE ST_Intersects(heat_tot_curr_density.rast,subAreas.soustracteGeom) group by subAreas.gid), " +\ + "statLoadProfilBySubarea as (select stat.load_profile.nuts_id as load_profile_nutsid, stat.load_profile.value as val_load_profile, " +\ + "stat.time.month as month_of_year, " +\ + "stat.time.hour_of_year as hour_of_year, " +\ + "stat.time.day as day_of_month, " +\ + "stat.time.hour_of_day as hour_of_day, " +\ + "statBySubAreas.count as statCount, statBySubAreas.sum as statSum_HD " +\ + "from stat.load_profile " +\ + "left join statBySubAreas on statBySubAreas.gid = stat.load_profile.fk_nuts_gid " +\ + "inner join stat.time on stat.load_profile.fk_time_id = stat.time.id " +\ + "WHERE fk_nuts_gid is not null and fk_time_id is not null " +\ + "AND statBySubAreas.gid = stat.load_profile.fk_nuts_gid AND stat.time.year = " + str(year) + " " +\ + "order by stat.time.hour_of_year), " +\ + "totalLoadprofile as ( " +\ + "select sum(val_load_profile) as tot_load_profile,load_profile_nutsid " +\ + "from statLoadProfilBySubarea group by load_profile_nutsid) " +\ + "select sum(val_load_profile/tot_load_profile*statSum_HD) as normalizedCalutation,hour_of_year " +\ + "from statLoadProfilBySubarea " +\ + "inner join totalLoadprofile on statLoadProfilBySubarea.load_profile_nutsid = totalLoadprofile.load_profile_nutsid " +\ + "group by hour_of_year " +\ + "order by normalizedCalutation DESC;" + + return sql_query diff --git a/api/app/models/indicators.py b/api/app/models/indicators.py index b59d1ec4..8c778c1f 100644 --- a/api/app/models/indicators.py +++ b/api/app/models/indicators.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -from app.constants import nuts0, nuts1, nuts2, nuts3, lau2, hectare_name +from app.constants import hectare_name, lau2, nuts0, nuts1, nuts2, nuts3 # LAYERS diff --git a/api/app/models/indicators_bak.py b/api/app/models/indicators_bak.py index ac70ce41..a1076e7f 100644 --- a/api/app/models/indicators_bak.py +++ b/api/app/models/indicators_bak.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from app.constants import nuts0,nuts1,nuts2,nuts3,lau2,hectare_name +from app.constants import hectare_name, lau2, nuts0, nuts1, nuts2, nuts3 # LAYERS """ POPULATION_TOT = constants.POPULATION_TOT @@ -76,462 +76,461 @@ # ALL DATA FOR THE STATS layersData = { - HEAT_DENSITY_TOT:{'tablename':HEAT_DENSITY_TOT, - 'from_indicator_name':stat + HEAT_DENSITY_TOT, - 'where':'', + HEAT_DENSITY_TOT:{'tablename':HEAT_DENSITY_TOT, + 'from_indicator_name':stat + HEAT_DENSITY_TOT, + 'where':'', 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'crs': '3035', 'geo_column': geometry_column, - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'data_aggregated':True,'indicators':[ - {'table_column': 'sum', 'unit': 'MWh','indicator_id':'consumption','factor': 0.00001,}, - {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, - {'table_column': 'min', 'unit': 'MWh','indicator_id':'consumption_min'}, - {'table_column': 'max', 'unit': 'MWh','indicator_id':'consumption_max'}, - {'table_column': 'mean', 'unit': 'MWh/ha','indicator_id':'density'}, - { - 'reference_indicator_id_1': 'consumption','reference_tablename_indicator_id_1':HEAT_DENSITY_TOT, - 'operator': '/', - 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, - 'unit':'MWh/person', 'indicator_id':HEAT_DENSITY_TOT+'_per_'+POPULATION_TOT - }, - ]}, - HEAT_DENSITY_RES:{'tablename':HEAT_DENSITY_RES, - 'from_indicator_name':stat + HEAT_DENSITY_RES, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'data_aggregated':True,'indicators':[ + {'table_column': 'sum', 'unit': 'MWh','indicator_id':'consumption','factor': 0.00001,}, + {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, + {'table_column': 'min', 'unit': 'MWh','indicator_id':'consumption_min'}, + {'table_column': 'max', 'unit': 'MWh','indicator_id':'consumption_max'}, + {'table_column': 'mean', 'unit': 'MWh/ha','indicator_id':'density'}, + { + 'reference_indicator_id_1': 'consumption','reference_tablename_indicator_id_1':HEAT_DENSITY_TOT, + 'operator': '/', + 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, + 'unit':'MWh/person', 'indicator_id':HEAT_DENSITY_TOT+'_per_'+POPULATION_TOT + }, + ]}, + HEAT_DENSITY_RES:{'tablename':HEAT_DENSITY_RES, + 'from_indicator_name':stat + HEAT_DENSITY_RES, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - - 'data_aggregated':True,'indicators':[ - {'table_column': 'sum', 'unit': 'MWh','indicator_id':'consumption'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + + 'data_aggregated':True,'indicators':[ + {'table_column': 'sum', 'unit': 'MWh','indicator_id':'consumption'}, {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, {'table_column': 'mean', 'unit': 'MWh/ha','indicator_id':'density'}, - {'reference_indicator_id_1': 'consumption', 'reference_tablename_indicator_id_1':HEAT_DENSITY_RES, + {'reference_indicator_id_1': 'consumption', 'reference_tablename_indicator_id_1':HEAT_DENSITY_RES, 'operator': '/', - 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, + 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'MWh/person', 'indicator_id':HEAT_DENSITY_RES+'_per_'+POPULATION_TOT - }, - ] - }, - HEAT_DENSITY_NON_RES:{'tablename':HEAT_DENSITY_NON_RES, - 'from_indicator_name':stat + HEAT_DENSITY_NON_RES, + }, + ] + }, + HEAT_DENSITY_NON_RES:{'tablename':HEAT_DENSITY_NON_RES, + 'from_indicator_name':stat + HEAT_DENSITY_NON_RES, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - - 'data_aggregated':True,'indicators':[ - {'table_column': 'sum', 'unit': 'MWh','indicator_id':'consumption'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + + 'data_aggregated':True,'indicators':[ + {'table_column': 'sum', 'unit': 'MWh','indicator_id':'consumption'}, {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, {'table_column': 'mean', 'unit': 'MWh/ha','indicator_id':'density'}, - {'reference_indicator_id_1': 'consumption', 'reference_tablename_indicator_id_1':HEAT_DENSITY_NON_RES, + {'reference_indicator_id_1': 'consumption', 'reference_tablename_indicator_id_1':HEAT_DENSITY_NON_RES, 'operator': '/', - 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, + 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'MWh/person', 'indicator_id':HEAT_DENSITY_NON_RES+'_per_'+POPULATION_TOT - }, - ]}, - GRASS_FLOOR_AREA_TOT:{'tablename':GRASS_FLOOR_AREA_TOT, + }, + ]}, + GRASS_FLOOR_AREA_TOT:{'tablename':GRASS_FLOOR_AREA_TOT, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - - 'from_indicator_name':stat + GRASS_FLOOR_AREA_TOT, - 'data_aggregated':True,'indicators':[ - {'table_column': 'sum', 'unit': 'm2','indicator_id':'total'}, - {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + + 'from_indicator_name':stat + GRASS_FLOOR_AREA_TOT, + 'data_aggregated':True,'indicators':[ + {'table_column': 'sum', 'unit': 'm2','indicator_id':'total'}, + {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, {'table_column': 'mean', 'unit': 'm2/ha','indicator_id':'density'}, - {'reference_indicator_id_1': 'total', 'reference_tablename_indicator_id_1':GRASS_FLOOR_AREA_TOT, + {'reference_indicator_id_1': 'total', 'reference_tablename_indicator_id_1':GRASS_FLOOR_AREA_TOT, 'operator': '/', - 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, + 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'m2/person', 'indicator_id':GRASS_FLOOR_AREA_TOT+'_per_'+POPULATION_TOT - }, + }, - ] - }, - GRASS_FLOOR_AREA_RES:{'tablename':GRASS_FLOOR_AREA_RES, + ] + }, + GRASS_FLOOR_AREA_RES:{'tablename':GRASS_FLOOR_AREA_RES, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - - 'from_indicator_name':stat + GRASS_FLOOR_AREA_RES, - 'data_aggregated':True,'indicators':[ - {'table_column': 'sum', 'unit': 'm2','indicator_id':'total'}, - {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + + 'from_indicator_name':stat + GRASS_FLOOR_AREA_RES, + 'data_aggregated':True,'indicators':[ + {'table_column': 'sum', 'unit': 'm2','indicator_id':'total'}, + {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, {'table_column': 'mean', 'unit': 'm2/ha','indicator_id':'density'}, - {'reference_indicator_id_1': 'total', 'reference_tablename_indicator_id_1':GRASS_FLOOR_AREA_RES, + {'reference_indicator_id_1': 'total', 'reference_tablename_indicator_id_1':GRASS_FLOOR_AREA_RES, 'operator': '/', - 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, + 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'m2/person', 'indicator_id':GRASS_FLOOR_AREA_RES+'_per_'+POPULATION_TOT - }, - ] - }, - GRASS_FLOOR_AREA_NON_RES:{'tablename':GRASS_FLOOR_AREA_NON_RES, + }, + ] + }, + GRASS_FLOOR_AREA_NON_RES:{'tablename':GRASS_FLOOR_AREA_NON_RES, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - - 'from_indicator_name': stat + GRASS_FLOOR_AREA_NON_RES, - 'data_aggregated':True,'indicators':[ - {'table_column': 'sum', 'unit': 'm2','indicator_id':'total'}, - {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + + 'from_indicator_name': stat + GRASS_FLOOR_AREA_NON_RES, + 'data_aggregated':True,'indicators':[ + {'table_column': 'sum', 'unit': 'm2','indicator_id':'total'}, + {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, {'table_column': 'mean', 'unit': 'm2/ha','indicator_id':'density'}, - {'reference_indicator_id_1': 'total', 'reference_tablename_indicator_id_1':GRASS_FLOOR_AREA_NON_RES, + {'reference_indicator_id_1': 'total', 'reference_tablename_indicator_id_1':GRASS_FLOOR_AREA_NON_RES, 'operator': '/', - 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, + 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'m2/person', 'indicator_id':GRASS_FLOOR_AREA_NON_RES+'_per_'+POPULATION_TOT - }, - ] - }, - BUILDING_VOLUMES_TOT:{'tablename':BUILDING_VOLUMES_TOT, + }, + ] + }, + BUILDING_VOLUMES_TOT:{'tablename':BUILDING_VOLUMES_TOT, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'from_indicator_name':stat + BUILDING_VOLUMES_TOT, - 'data_aggregated':True,'indicators':[ - {'table_column': 'sum', 'unit': 'm3','indicator_id':'total'}, - {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'from_indicator_name':stat + BUILDING_VOLUMES_TOT, + 'data_aggregated':True,'indicators':[ + {'table_column': 'sum', 'unit': 'm3','indicator_id':'total'}, + {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, {'table_column': 'mean', 'unit': 'm3/ha','indicator_id':'density'}, - {'reference_indicator_id_1': 'total', 'reference_tablename_indicator_id_1':BUILDING_VOLUMES_TOT, + {'reference_indicator_id_1': 'total', 'reference_tablename_indicator_id_1':BUILDING_VOLUMES_TOT, 'operator': '/', - 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, + 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'m3/person', 'indicator_id':BUILDING_VOLUMES_TOT+'_per_'+POPULATION_TOT - }, - ] - }, - BUILDING_VOLUMES_RES:{'tablename':BUILDING_VOLUMES_RES, + }, + ] + }, + BUILDING_VOLUMES_RES:{'tablename':BUILDING_VOLUMES_RES, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'from_indicator_name':stat + BUILDING_VOLUMES_RES, - 'data_aggregated':True,'indicators':[ - {'table_column': 'sum', 'unit': 'm3','indicator_id':'total'}, - {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'from_indicator_name':stat + BUILDING_VOLUMES_RES, + 'data_aggregated':True,'indicators':[ + {'table_column': 'sum', 'unit': 'm3','indicator_id':'total'}, + {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, {'table_column': 'mean', 'unit': 'm3/ha','indicator_id':'density'}, - {'reference_indicator_id_1': 'total', 'reference_tablename_indicator_id_1':BUILDING_VOLUMES_RES, + {'reference_indicator_id_1': 'total', 'reference_tablename_indicator_id_1':BUILDING_VOLUMES_RES, 'operator': '/', - 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, + 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'m3/person', 'indicator_id':BUILDING_VOLUMES_RES+'_per_'+POPULATION_TOT - }, - ] - }, - BUILDING_VOLUMES_NON_RES:{'tablename':BUILDING_VOLUMES_NON_RES, - 'from_indicator_name':stat + BUILDING_VOLUMES_NON_RES, + }, + ] + }, + BUILDING_VOLUMES_NON_RES:{'tablename':BUILDING_VOLUMES_NON_RES, + 'from_indicator_name':stat + BUILDING_VOLUMES_NON_RES, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'data_aggregated':True,'indicators':[ - {'table_column': 'sum', 'unit': 'm3','indicator_id':'total'}, - {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'data_aggregated':True,'indicators':[ + {'table_column': 'sum', 'unit': 'm3','indicator_id':'total'}, + {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, {'table_column': 'mean', 'unit': 'm3/ha','indicator_id':'density'}, - {'reference_indicator_id_1': 'total', 'reference_tablename_indicator_id_1':BUILDING_VOLUMES_NON_RES, + {'reference_indicator_id_1': 'total', 'reference_tablename_indicator_id_1':BUILDING_VOLUMES_NON_RES, 'operator': '/', - 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, + 'reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'m3/person', 'indicator_id':BUILDING_VOLUMES_NON_RES+'_per_'+POPULATION_TOT - }, + }, ] - }, - INDUSTRIAL_SITES_EMISSIONS:{'tablename':INDUSTRIAL_SITES_EMISSIONS, - 'from_indicator_name':stat + INDUSTRIAL_SITES_EMISSIONS, + }, + INDUSTRIAL_SITES_EMISSIONS:{'tablename':INDUSTRIAL_SITES_EMISSIONS, + 'from_indicator_name':stat + INDUSTRIAL_SITES_EMISSIONS, 'schema_scalelvl': public_schema, 'schema_hectare': public_schema, 'geo_column': geom_column, 'crs': '4326', - 'table_type':vector_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'data_aggregated':False, - 'indicators':[ - {'table_column': 'emissions_ets_2014', 'unit': 'Tonnes/year','indicator_id':'value'}, - ] - }, - INDUSTRIAL_SITES_EXCESS_HEAT:{'tablename':INDUSTRIAL_SITES_EXCESS_HEAT, - 'from_indicator_name':stat + INDUSTRIAL_SITES_EXCESS_HEAT, + 'table_type':vector_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'data_aggregated':False, + 'indicators':[ + {'table_column': 'emissions_ets_2014', 'unit': 'Tonnes/year','indicator_id':'value'}, + ] + }, + INDUSTRIAL_SITES_EXCESS_HEAT:{'tablename':INDUSTRIAL_SITES_EXCESS_HEAT, + 'from_indicator_name':stat + INDUSTRIAL_SITES_EXCESS_HEAT, 'schema_scalelvl': public_schema, 'schema_hectare': public_schema, 'geo_column': geom_column, 'crs': '4326', - 'table_type':vector_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - - 'data_aggregated':False,'indicators':[ - {'table_column': 'excess_heat_100_200c', 'unit': 'GWh/year','indicator_id':'value1'}, - {'table_column': 'excess_heat_200_500c', 'unit': 'GWh/year','indicator_id':'value2'}, - {'table_column': 'excess_heat_500c', 'unit': 'GWh/year','indicator_id':'value3'}, - {'table_column': 'excess_heat_total', 'unit': 'GWh/year','indicator_id':'total'} - ] - }, - POPULATION_TOT:{ - 'tablename':POPULATION_TOT, - 'from_indicator_name':stat + POPULATION_TOT, + 'table_type':vector_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + + 'data_aggregated':False,'indicators':[ + {'table_column': 'excess_heat_100_200c', 'unit': 'GWh/year','indicator_id':'value1'}, + {'table_column': 'excess_heat_200_500c', 'unit': 'GWh/year','indicator_id':'value2'}, + {'table_column': 'excess_heat_500c', 'unit': 'GWh/year','indicator_id':'value3'}, + {'table_column': 'excess_heat_total', 'unit': 'GWh/year','indicator_id':'total'} + ] + }, + POPULATION_TOT:{ + 'tablename':POPULATION_TOT, + 'from_indicator_name':stat + POPULATION_TOT, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], 'geo_column': geometry_column, - 'data_aggregated':True,'indicators':[ - {'table_column': 'sum', 'unit': 'person','indicator_id':'population'}, - {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, - {'reference_indicator_id_1': 'population','reference_tablename_indicator_id_1':POPULATION_TOT, 'operator': '/','reference_indicator_id_2':'count_cell','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'person/ha', 'indicator_id':'density'}, - ] - }, + 'data_aggregated':True,'indicators':[ + {'table_column': 'sum', 'unit': 'person','indicator_id':'population'}, + {'table_column': 'count', 'unit': 'cells','indicator_id':'count_cell'}, + {'reference_indicator_id_1': 'population','reference_tablename_indicator_id_1':POPULATION_TOT, 'operator': '/','reference_indicator_id_2':'count_cell','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'person/ha', 'indicator_id':'density'}, + ] + }, WWTP:{'tablename':WWTP, - 'from_indicator_name':stat + WWTP, + 'from_indicator_name':stat + WWTP, 'schema_scalelvl': geo_schema, 'schema_hectare': geo_schema, 'geo_column': geom_column, 'crs': '3035', - 'table_type':vector_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'data_aggregated':False, - 'indicators':[ - {'table_column': 'capacity', 'unit': 'kW','indicator_id':'power'}, - {'table_column': 'power', 'unit': 'Person equivalent','indicator_id':'capacity'}, - ] - }, + 'table_type':vector_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'data_aggregated':False, + 'indicators':[ + {'table_column': 'capacity', 'unit': 'kW','indicator_id':'power'}, + {'table_column': 'power', 'unit': 'Person equivalent','indicator_id':'capacity'}, + ] + }, WWTP_CAPACITY:{'tablename':WWTP_CAPACITY, 'schema_scalelvl': public_schema, 'schema_hectare': public_schema, 'crs': '3035', - 'table_type':vector_type, - 'from_indicator_name':stat + WWTP_CAPACITY, + 'table_type':vector_type, + 'from_indicator_name':stat + WWTP_CAPACITY, 'geo_column': geometry_column, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - - 'data_aggregated':False, - 'indicators':[ - {'table_column': 'capacity', 'unit': 'Person equivalent','indicator_id':'capacity'}, - ]}, - WWTP_POWER:{'tablename':WWTP_POWER, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + + 'data_aggregated':False, + 'indicators':[ + {'table_column': 'capacity', 'unit': 'Person equivalent','indicator_id':'capacity'}, + ]}, + WWTP_POWER:{'tablename':WWTP_POWER, 'schema_scalelvl': public_schema, 'schema_hectare': public_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':vector_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'from_indicator_name':stat + WWTP_POWER, - 'data_aggregated':False, - 'indicators':[ - {'table_column': 'power', 'unit': 'kW','indicator_id':'power'}, - ] - }, + 'table_type':vector_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'from_indicator_name':stat + WWTP_POWER, + 'data_aggregated':False, + 'indicators':[ + {'table_column': 'power', 'unit': 'kW','indicator_id':'power'}, + ] + }, LIVESTOCK_EFFLUENTS:{'tablename':LIVESTOCK_EFFLUENTS, - 'from_indicator_name':stat + LIVESTOCK_EFFLUENTS, - 'where':'livestock_effluents', + 'from_indicator_name':stat + LIVESTOCK_EFFLUENTS, + 'where':'livestock_effluents', 'schema_scalelvl': geo_schema, 'schema_hectare': geo_schema, 'crs': '3035', 'geo_column': geometry_column, - 'table_type':vector_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'scalelvl_column':'code', - 'data_aggregated':True,'indicators':[ - {'table_column': 'value', 'unit': 'PJ','indicator_id':'NUTS_potential'}, + 'table_type':vector_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'scalelvl_column':'code', + 'data_aggregated':True,'indicators':[ + {'table_column': 'value', 'unit': 'PJ','indicator_id':'NUTS_potential'}, {'reference_indicator_id_1': 'NUTS_potential', 'reference_tablename_indicator_id_1':LIVESTOCK_EFFLUENTS, 'operator': '/','reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'PJ/person', 'indicator_id':'Livestock_effl_pp'} - - ]}, + + ]}, AGRICULTURAL_RESIDUES:{'tablename':AGRICULTURAL_RESIDUES, - 'from_indicator_name':stat + AGRICULTURAL_RESIDUES, - 'where':'livestock_effluents', + 'from_indicator_name':stat + AGRICULTURAL_RESIDUES, + 'where':'livestock_effluents', 'schema_scalelvl': geo_schema, 'schema_hectare': geo_schema, 'crs': '3035', 'geo_column': geometry_column, - 'table_type':vector_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'scalelvl_column':'code', - 'data_aggregated':True,'indicators':[ - {'table_column': 'value', 'unit': 'PJ','indicator_id':'NUTS_potential'}, - {'reference_indicator_id_1': 'NUTS_potential', 'reference_tablename_indicator_id_1':AGRICULTURAL_RESIDUES, 'operator': '/','reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'PJ/person', 'indicator_id':'agriculture_pp'} - ]}, - POTENTIAL_FOREST:{'tablename':POTENTIAL_FOREST, + 'table_type':vector_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'scalelvl_column':'code', + 'data_aggregated':True,'indicators':[ + {'table_column': 'value', 'unit': 'PJ','indicator_id':'NUTS_potential'}, + {'reference_indicator_id_1': 'NUTS_potential', 'reference_tablename_indicator_id_1':AGRICULTURAL_RESIDUES, 'operator': '/','reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'PJ/person', 'indicator_id':'agriculture_pp'} + ]}, + POTENTIAL_FOREST:{'tablename':POTENTIAL_FOREST, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'from_indicator_name':stat + POTENTIAL_FOREST, - 'data_aggregated':True,'indicators':[ - {'table_column': 'mean', 'unit': 'PJ/ha','indicator_id':'average'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'from_indicator_name':stat + POTENTIAL_FOREST, + 'data_aggregated':True,'indicators':[ + {'table_column': 'mean', 'unit': 'PJ/ha','indicator_id':'average'}, {'table_column': 'sum', 'unit': 'PJ/ha x cells','indicator_id':'value'}, - {'table_column': 'count', 'unit': 'cells','indicator_id':'cells'}, - ] - }, - MUNICIPAL_SOLID_WASTE:{'tablename':MUNICIPAL_SOLID_WASTE, - 'from_indicator_name':stat + MUNICIPAL_SOLID_WASTE, - 'where':'', + {'table_column': 'count', 'unit': 'cells','indicator_id':'cells'}, + ] + }, + MUNICIPAL_SOLID_WASTE:{'tablename':MUNICIPAL_SOLID_WASTE, + 'from_indicator_name':stat + MUNICIPAL_SOLID_WASTE, + 'where':'', 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'crs': '3035', 'geo_column': geometry_column, - 'table_type':vector_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'scalelvl_column':'code', - 'data_aggregated':True,'indicators':[ - {'table_column': 'value', 'unit': 'PJ','indicator_id':'val'}, - {'reference_indicator_id_1': 'val', 'reference_tablename_indicator_id_1':MUNICIPAL_SOLID_WASTE, 'operator': '/','reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'PJ/person', 'indicator_id':'agriculture_pp'}, - ]}, - GEOTHERMAL_POTENTIAL_HEAT_COND:{'tablename':GEOTHERMAL_POTENTIAL_HEAT_COND, - 'from_indicator_name':stat + GEOTHERMAL_POTENTIAL_HEAT_COND, + 'table_type':vector_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'scalelvl_column':'code', + 'data_aggregated':True,'indicators':[ + {'table_column': 'value', 'unit': 'PJ','indicator_id':'val'}, + {'reference_indicator_id_1': 'val', 'reference_tablename_indicator_id_1':MUNICIPAL_SOLID_WASTE, 'operator': '/','reference_indicator_id_2':'population','reference_tablename_indicator_id_2':POPULATION_TOT, 'unit':'PJ/person', 'indicator_id':'agriculture_pp'}, + ]}, + GEOTHERMAL_POTENTIAL_HEAT_COND:{'tablename':GEOTHERMAL_POTENTIAL_HEAT_COND, + 'from_indicator_name':stat + GEOTHERMAL_POTENTIAL_HEAT_COND, 'schema_scalelvl': geo_schema, 'schema_hectare': geo_schema, 'geo_column': geom_column, 'crs': '4326', - 'table_type':vector_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'data_aggregated':False,'indicators':[ - {'table_column': 'sum', 'unit': 'W/mK','indicator_id':'value'} - ] - }, - SOLAR_POTENTIAL:{'tablename':SOLAR_POTENTIAL, - 'from_indicator_name':stat + SOLAR_POTENTIAL, + 'table_type':vector_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'data_aggregated':False,'indicators':[ + {'table_column': 'sum', 'unit': 'W/mK','indicator_id':'value'} + ] + }, + SOLAR_POTENTIAL:{'tablename':SOLAR_POTENTIAL, + 'from_indicator_name':stat + SOLAR_POTENTIAL, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'data_aggregated':True, - 'indicators':[ + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'data_aggregated':True, + 'indicators':[ {'table_column': 'mean', 'unit': 'kWh/m^2','indicator_id':'average'}, {'table_column': 'min', 'unit': 'kWh/m^2','indicator_id':'min'}, {'table_column': 'max', 'unit': 'kWh/m^2','indicator_id':'max'}, - {'table_column': 'count', 'unit': 'cells','indicator_id':'cells'}, - ] - }, - WIND_POTENTIAL:{'tablename':WIND_POTENTIAL, + {'table_column': 'count', 'unit': 'cells','indicator_id':'cells'}, + ] + }, + WIND_POTENTIAL:{'tablename':WIND_POTENTIAL, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'from_indicator_name':stat + WIND_POTENTIAL, - 'data_aggregated':True, - 'indicators':[ - {'table_column': 'mean', 'unit': 'm/s','indicator_id':'average'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'from_indicator_name':stat + WIND_POTENTIAL, + 'data_aggregated':True, + 'indicators':[ + {'table_column': 'mean', 'unit': 'm/s','indicator_id':'average'}, {'table_column': 'max', 'unit': 'm/s','indicator_id':'max'}, {'table_column': 'min', 'unit': 'm/s','indicator_id':'min'}, {'table_column': 'count', 'unit': 'cells','indicator_id':'cells'} - ] - }, - HDD_CUR:{'tablename':HDD_CUR, + ] + }, + HDD_CUR:{'tablename':HDD_CUR, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'from_indicator_name':stat + HDD_CUR, - 'data_aggregated':True, - 'indicators':[ - {'table_column': 'mean', 'unit': 'Kd','indicator_id':'average'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'from_indicator_name':stat + HDD_CUR, + 'data_aggregated':True, + 'indicators':[ + {'table_column': 'mean', 'unit': 'Kd','indicator_id':'average'}, {'table_column': 'max', 'unit': 'Kd','indicator_id':'max'}, {'table_column': 'min', 'unit': 'Kd','indicator_id':'min'}, {'table_column': 'count', 'unit': 'cells','indicator_id':'cells'} - ] - }, - CDD_CUR:{ - 'tablename':CDD_CUR, + ] + }, + CDD_CUR:{ + 'tablename':CDD_CUR, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'from_indicator_name':stat + CDD_CUR, - 'data_aggregated':True, - 'indicators':[ - {'table_column': 'mean', 'unit': 'Kd','indicator_id':'average'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'from_indicator_name':stat + CDD_CUR, + 'data_aggregated':True, + 'indicators':[ + {'table_column': 'mean', 'unit': 'Kd','indicator_id':'average'}, {'table_column': 'max', 'unit': 'Kd','indicator_id':'max'}, {'table_column': 'min', 'unit': 'Kd','indicator_id':'min'}, {'table_column': 'count', 'unit': 'cells','indicator_id':'cells'} - ]}, + ]}, LAND_SURFACE_TEMP:{ 'tablename':LAND_SURFACE_TEMP, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'from_indicator_name':stat + LAND_SURFACE_TEMP, - 'data_aggregated':True, - 'indicators':[ - {'table_column': 'count', 'unit': 'cells','indicator_id':'cells'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'from_indicator_name':stat + LAND_SURFACE_TEMP, + 'data_aggregated':True, + 'indicators':[ + {'table_column': 'count', 'unit': 'cells','indicator_id':'cells'}, {'table_column': 'min', 'unit': 'degree C','indicator_id':'minimum'}, {'table_column': 'max', 'unit': 'degree C','indicator_id':'maximum'}, {'table_column': 'mean', 'unit': 'degree C','indicator_id':'average'} - ] - }, - SOLAR_RADIATION:{'tablename':SOLAR_RADIATION, - 'from_indicator_name':stat + SOLAR_RADIATION, - 'where':'', + ] + }, + SOLAR_RADIATION:{'tablename':SOLAR_RADIATION, + 'from_indicator_name':stat + SOLAR_RADIATION, + 'where':'', 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'crs': '3035', 'geo_column': geometry_column, - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'data_aggregated':True,'indicators':[ + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'data_aggregated':True,'indicators':[ {'table_column': 'mean', 'unit': 'kWh/m^2','indicator_id':'average'}, {'table_column': 'min', 'unit': 'kWh/m^2','indicator_id':'min'}, {'table_column': 'max', 'unit': 'kWh/m^2','indicator_id':'max'}, - {'table_column': 'count', 'unit': 'cells','indicator_id':'cells'}, - ]}, - WIND_SPEED:{'tablename':WIND_SPEED, + {'table_column': 'count', 'unit': 'cells','indicator_id':'cells'}, + ]}, + WIND_SPEED:{'tablename':WIND_SPEED, 'schema_scalelvl': stat_schema, 'schema_hectare': geo_schema, 'geo_column': geometry_column, 'crs': '3035', - 'table_type':raster_type, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'from_indicator_name':stat + WIND_SPEED, - 'data_aggregated':True, - 'indicators':[ - {'table_column': 'mean', 'unit': 'm/s','indicator_id':'average'}, + 'table_type':raster_type, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'from_indicator_name':stat + WIND_SPEED, + 'data_aggregated':True, + 'indicators':[ + {'table_column': 'mean', 'unit': 'm/s','indicator_id':'average'}, {'table_column': 'max', 'unit': 'm/s','indicator_id':'max'}, {'table_column': 'min', 'unit': 'm/s','indicator_id':'min'}, {'table_column': 'count', 'unit': 'cells','indicator_id':'cells'} - ] - }, + ] + }, ELECTRICITY_CO2_EMISSION_FACTOR:{'tablename':ELECTRICITY_CO2_EMISSION_FACTOR, 'schema_scalelvl': public_schema, 'schema_hectare': public_schema, 'geo_column': geom_column, 'crs': '4258', - 'table_type':vector_type, - 'level_of_data':'NUTS 0', - 'from_indicator_name':stat + ELECTRICITY_CO2_EMISSION_FACTOR, - 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], - 'data_aggregated':True,'indicators':[ - {'table_column': 'value', 'unit': 'kg/MWh','indicator_id':'density'} - ] - }, + 'table_type':vector_type, + 'level_of_data':'NUTS 0', + 'from_indicator_name':stat + ELECTRICITY_CO2_EMISSION_FACTOR, + 'data_lvl':[nuts0,nuts1,nuts2,nuts3,lau2,hectare_name], + 'data_aggregated':True,'indicators':[ + {'table_column': 'value', 'unit': 'kg/MWh','indicator_id':'density'} + ] + }, } - diff --git a/api/app/models/lau.py b/api/app/models/lau.py index f87c3ae7..7784103b 100644 --- a/api/app/models/lau.py +++ b/api/app/models/lau.py @@ -2,9 +2,9 @@ from app import dbGIS as db from geoalchemy2 import Geometry -from sqlalchemy import func -from geojson import FeatureCollection, Feature from geoalchemy2.shape import to_shape +from geojson import Feature, FeatureCollection +from sqlalchemy import func class Lau(db.Model): @@ -27,5 +27,3 @@ class Lau(db.Model): def __repr__(self): return "" % ( self.comm_id, self.stat_levl_) - - diff --git a/api/app/models/nuts.py b/api/app/models/nuts.py index c622ecb6..64b880e0 100644 --- a/api/app/models/nuts.py +++ b/api/app/models/nuts.py @@ -2,9 +2,9 @@ from app import dbGIS as db from geoalchemy2 import Geometry -from sqlalchemy import func -from geojson import FeatureCollection, Feature from geoalchemy2.shape import to_shape +from geojson import Feature, FeatureCollection +from sqlalchemy import func class Nuts(db.Model): diff --git a/api/app/models/population_density.py b/api/app/models/population_density.py index eb17b2ae..b54ad5da 100644 --- a/api/app/models/population_density.py +++ b/api/app/models/population_density.py @@ -1,6 +1,9 @@ +from decimal import * + from app import dbGIS as db from geoalchemy2 import Raster -from decimal import * + + """ Population Density layer as ha """ @@ -92,4 +95,3 @@ class PopulationDensityNutsModel(db.Model): def __repr__(self): str_date = self.date.strftime("%Y-%m-%d") return "" % (self.nuts_id, str_date, self.value, str(self.nuts)) - diff --git a/api/app/models/role.py b/api/app/models/role.py index 62ccd7f7..93648e25 100644 --- a/api/app/models/role.py +++ b/api/app/models/role.py @@ -1,6 +1,8 @@ -from .. import dbGIS as db from flask_security import RoleMixin +from .. import dbGIS as db + + class Role(db.Model, RoleMixin): ''' The model for a user in the database @@ -14,4 +16,3 @@ class Role(db.Model, RoleMixin): id = db.Column(db.Integer(), primary_key=True) name = db.Column(db.String(80), unique=True) description = db.Column(db.String(255)) - diff --git a/api/app/models/snapshots.py b/api/app/models/snapshots.py index ad208f2c..310fbbe4 100644 --- a/api/app/models/snapshots.py +++ b/api/app/models/snapshots.py @@ -1,5 +1,6 @@ from .. import dbGIS as db -from ..decorators.restplus import UserUnidentifiedException, ParameterException, RequestException +from ..decorators.restplus import (ParameterException, RequestException, + UserUnidentifiedException) class Snapshots(db.Model): diff --git a/api/app/models/statsQueries.py b/api/app/models/statsQueries.py index 8c907660..1cfd8ba8 100644 --- a/api/app/models/statsQueries.py +++ b/api/app/models/statsQueries.py @@ -1,162 +1,162 @@ import datetime -from .. import helper -from app import dbGIS as db -from app import constants +import logging from decimal import * -from app.models.indicators import layersData, ELECRICITY_MIX -from app import celery -from . import generalData + +from app import celery, constants +from app import dbGIS as db from app import model +from app.models.indicators import ELECRICITY_MIX, layersData + +from .. import helper +from . import generalData -import logging log = logging.getLogger(__name__) class LayersStats: - @staticmethod - def run_stat(payload): - - year = payload['year'] - - layersPayload = payload['layers'] - - scale_level = payload['scale_level'] - - - #must sanitize this - - selection_areas = '' - is_hectare = False - noDataLayers=[] - layers=[] - output=[] - - if scale_level in constants.NUTS_LAU_VALUES: - selection_areas = payload['nuts'] - - elif scale_level == constants.hectare_name: - selection_areas = payload['areas'] - geom = helper.areas_to_geom(selection_areas) - is_hectare=True - - for c, layer in enumerate(layersPayload): - if layersPayload[c] in layersData: - layers.append(layersPayload[c]) - else: - noDataLayers.append(layersPayload[c]) - - - if is_hectare: - output = LayersStats.get_stats(selection_areas=geom, year=year, layers=layers,scale_level=scale_level, is_hectare=is_hectare) - else: - nuts = ''.join("'"+str(nu)+"'," for nu in selection_areas)[:-1] - output = LayersStats.get_stats(selection_areas=nuts, year=year, layers=layers, scale_level=scale_level, is_hectare=False) - - return output, noDataLayers - - @staticmethod - def get_stats(year, layers, selection_areas, is_hectare, scale_level): - # Get the number of layers - result = [] - # Check if there is at least one layer - if layers: - # Construction of the query - sql_query = '' - sql_with = ' WITH ' - sql_select = ' SELECT ' - sql_from = ' FROM ' - for layer in layers: - - if len(layersData[layer]['indicators']) != 0 and scale_level in layersData[layer]['data_lvl']: - if is_hectare: - sql_with += generalData.constructWithPartEachLayerHectare(geometry=selection_areas, year=year, layer=layer, scale_level=scale_level) + ',' - else: - sql_with += generalData.constructWithPartEachLayerNutsLau(layer=layer, nuts=selection_areas, year=year, scale_level=scale_level) + ',' - - for indicator in layersData[layer]['indicators']: - if 'table_column' in indicator: - sql_select += layer+indicator['indicator_id']+',' - elif indicator['reference_tablename_indicator_id_1'] in layers and indicator['reference_tablename_indicator_id_2'] in layers: - sql_select+= indicator['reference_tablename_indicator_id_1']+indicator['reference_indicator_id_1']+' '+indicator['operator']+' '+indicator['reference_tablename_indicator_id_2']+indicator['reference_indicator_id_2']+',' - sql_from += layersData[layer]['from_indicator_name']+',' - - - - # Combine string to a single query - sql_with = sql_with[:-1] - sql_select = sql_select[:-1] - sql_from = sql_from[:-1] - sql_query = sql_with + sql_select + sql_from + ';' - - - # Run the query - query_geographic_database_first = model.query_geographic_database_first(sql_query) - - # Storing the results only if there is data - count_indic = 0 - areas = selection_areas.split(",") - for layer in layers: - values = [] - for indicator in layersData[layer]['indicators']: - if ('table_column' not in indicator and (indicator['reference_tablename_indicator_id_1'] not in layers or indicator['reference_tablename_indicator_id_2'] not in layers)) or scale_level not in layersData[layer]['data_lvl']: - continue - - currentValue = query_geographic_database_first[count_indic] or 0 - count_indic += 1 - - if "agg_method" in indicator and indicator["agg_method"] == "mean": - currentValue /= len(areas) - - if 'factor' in indicator: # Decimal * float => rise error - currentValue = float(currentValue) * float(indicator['factor']) - - try: - values.append({ - 'name': layer + '_' + indicator['indicator_id'], - 'value': currentValue, - 'unit': indicator['unit'] - }) - except KeyError: # Special case we retrieve only one value for an hectare - pass - result.append({ - 'name': layer, - 'values': values - }) - - return result + @staticmethod + def run_stat(payload): + + year = payload['year'] + + layersPayload = payload['layers'] + + scale_level = payload['scale_level'] + + + #must sanitize this + + selection_areas = '' + is_hectare = False + noDataLayers=[] + layers=[] + output=[] + + if scale_level in constants.NUTS_LAU_VALUES: + selection_areas = payload['nuts'] + + elif scale_level == constants.hectare_name: + selection_areas = payload['areas'] + geom = helper.areas_to_geom(selection_areas) + is_hectare=True + + for c, layer in enumerate(layersPayload): + if layersPayload[c] in layersData: + layers.append(layersPayload[c]) + else: + noDataLayers.append(layersPayload[c]) + + + if is_hectare: + output = LayersStats.get_stats(selection_areas=geom, year=year, layers=layers,scale_level=scale_level, is_hectare=is_hectare) + else: + nuts = ''.join("'"+str(nu)+"'," for nu in selection_areas)[:-1] + output = LayersStats.get_stats(selection_areas=nuts, year=year, layers=layers, scale_level=scale_level, is_hectare=False) + + return output, noDataLayers + + @staticmethod + def get_stats(year, layers, selection_areas, is_hectare, scale_level): + # Get the number of layers + result = [] + # Check if there is at least one layer + if layers: + # Construction of the query + sql_query = '' + sql_with = ' WITH ' + sql_select = ' SELECT ' + sql_from = ' FROM ' + for layer in layers: + + if len(layersData[layer]['indicators']) != 0 and scale_level in layersData[layer]['data_lvl']: + if is_hectare: + sql_with += generalData.constructWithPartEachLayerHectare(geometry=selection_areas, year=year, layer=layer, scale_level=scale_level) + ',' + else: + sql_with += generalData.constructWithPartEachLayerNutsLau(layer=layer, nuts=selection_areas, year=year, scale_level=scale_level) + ',' + + for indicator in layersData[layer]['indicators']: + if 'table_column' in indicator: + sql_select += layer+indicator['indicator_id']+',' + elif indicator['reference_tablename_indicator_id_1'] in layers and indicator['reference_tablename_indicator_id_2'] in layers: + sql_select+= indicator['reference_tablename_indicator_id_1']+indicator['reference_indicator_id_1']+' '+indicator['operator']+' '+indicator['reference_tablename_indicator_id_2']+indicator['reference_indicator_id_2']+',' + sql_from += layersData[layer]['from_indicator_name']+',' + + + + # Combine string to a single query + sql_with = sql_with[:-1] + sql_select = sql_select[:-1] + sql_from = sql_from[:-1] + sql_query = sql_with + sql_select + sql_from + ';' + + + # Run the query + query_geographic_database_first = model.query_geographic_database_first(sql_query) + + # Storing the results only if there is data + count_indic = 0 + areas = selection_areas.split(",") + for layer in layers: + values = [] + for indicator in layersData[layer]['indicators']: + if ('table_column' not in indicator and (indicator['reference_tablename_indicator_id_1'] not in layers or indicator['reference_tablename_indicator_id_2'] not in layers)) or scale_level not in layersData[layer]['data_lvl']: + continue + + currentValue = query_geographic_database_first[count_indic] or 0 + count_indic += 1 + + if "agg_method" in indicator and indicator["agg_method"] == "mean": + currentValue /= len(areas) + + if 'factor' in indicator: # Decimal * float => rise error + currentValue = float(currentValue) * float(indicator['factor']) + + try: + values.append({ + 'name': layer + '_' + indicator['indicator_id'], + 'value': currentValue, + 'unit': indicator['unit'] + }) + except KeyError: # Special case we retrieve only one value for an hectare + pass + result.append({ + 'name': layer, + 'values': values + }) + + return result class ElectricityMix: - @staticmethod - - def getEnergyMixNutsLau(nuts): + @staticmethod - sql_query = "WITH energy_total as (SELECT sum(electricity_generation) as value FROM " + ELECRICITY_MIX + " WHERE nuts0_code IN ("+nuts+") )" + \ - "SELECT DISTINCT energy_carrier, SUM(electricity_generation * 100 /energy_total.value) FROM " + ELECRICITY_MIX + " ,energy_total WHERE nuts0_code IN ("+nuts+") GROUP BY energy_carrier ORDER BY energy_carrier ASC" ; + def getEnergyMixNutsLau(nuts): + sql_query = "WITH energy_total as (SELECT sum(electricity_generation) as value FROM " + ELECRICITY_MIX + " WHERE nuts0_code IN ("+nuts+") )" + \ + "SELECT DISTINCT energy_carrier, SUM(electricity_generation * 100 /energy_total.value) FROM " + ELECRICITY_MIX + " ,energy_total WHERE nuts0_code IN ("+nuts+") GROUP BY energy_carrier ORDER BY energy_carrier ASC" ; - query = model.query_geographic_database(sql_query) - labels = [] - data = [] - backgroundColor = [] + query = model.query_geographic_database(sql_query) - for c, l in enumerate(query): + labels = [] + data = [] + backgroundColor = [] - labels.append(l[0]) - data.append(helper.roundValue(l[1])) - backgroundColor.append(helper.getGenerationMixColor(l[0])) - datasets = { - 'data' : data, - 'label': '%', - 'backgroundColor': backgroundColor - } + for c, l in enumerate(query): - result = { - 'labels':labels, - 'datasets':datasets - } - return result + labels.append(l[0]) + data.append(helper.roundValue(l[1])) + backgroundColor.append(helper.getGenerationMixColor(l[0])) + datasets = { + 'data' : data, + 'label': '%', + 'backgroundColor': backgroundColor + } + result = { + 'labels':labels, + 'datasets':datasets + } + return result diff --git a/api/app/models/uploads.py b/api/app/models/uploads.py index cc3c2ff9..21e7c7a8 100644 --- a/api/app/models/uploads.py +++ b/api/app/models/uploads.py @@ -1,684 +1,683 @@ -import csv -import json -import os -import shutil -import uuid -from functools import partial -from io import StringIO - -import pandas as pd -from pandas import DataFrame -import pyproj -import requests -import shapely.geometry as shapely_geom -import shapely.wkt as shapely_wkt - -import app.helper -from app import celery -from app import model -from geojson import Feature, FeatureCollection -from shapely.ops import transform -import xml.etree.ElementTree as ET -from urllib.parse import urlparse, parse_qs -import re - -from .. import constants, dbGIS as db -from ..decorators.exceptions import RequestException -from .. import helper - - -ALLOWED_EXTENSIONS = set(['tif', 'csv']) -GREATER_OR_EQUAL = 'greaterOrEqual' -GREATER = 'greater' -LESSER_OR_EQUAL = 'lesserOrEqual' -LESSER = 'lesser' -EQUAL = 'equal' - -class Uploads(db.Model): - ''' - This class will describe the model of a file uploaded by a user - ''' - __tablename__ = 'uploads' - __table_args__ = ( - {"schema": 'user'} - ) - - id = db.Column(db.Integer, primary_key=True) - uuid = db.Column(db.String(255)) - name = db.Column(db.String(255)) - layer = db.Column(db.String(255)) - layer_type = db.Column(db.String(255)) - size = db.Column(db.Numeric) - url = db.Column(db.String(255)) - is_generated = db.Column(db.Integer) - user_id = db.Column(db.Integer, db.ForeignKey('user.users.id')) - - -@celery.task(name='generate_tiles_file_upload') -def generate_tiles(upload_folder, grey_tif, layer_type, upload_uuid, user_currently_used_space): - ''' - This function is used to generate the various tiles of a layer in the db. - :param upload_folder: the folder of the upload - :param grey_tif: the url to the input file - :param layer_type: the type of the layer chosen for the input - :param upload_uuid: the uuid of the upload - :param user_currently_used_space: the space currently used by the user - - ''' - # we set up the directory for the tif - directory_for_tiles = upload_folder + '/tiles' - - tile_path = directory_for_tiles - access_rights = 0o755 - try: - os.mkdir(tile_path, access_rights) - except OSError: - print ("Creation of the directory %s failed" % tile_path) - else: - print ("Successfully created the directory %s" % tile_path) - - rgb_tif = upload_folder + '/rgba.tif' - if layer_type != 'custom': - helper.colorize(layer_type, grey_tif, rgb_tif) - else: - args_gdal = app.helper.commands_in_array("gdal_translate -of GTiff -expand rgba {} {} -co COMPRESS=DEFLATE ".format(grey_tif, rgb_tif)) - app.helper.run_command(args_gdal) - - try: - # commands launch to obtain the level of zooms - args_tiles = app.helper.commands_in_array("python3 app/helper/gdal2tiles.py -p 'mercator' -s 'EPSG:3035' -w 'leaflet' -r 'average' -z '4-11' {} {} ".format(rgb_tif, tile_path)) - app.helper.run_command(args_tiles) - - except : - generate_state = 10 - else: - generate_state = 0 - - # updating generate state of upload - upload = Uploads.query.filter_by(url=grey_tif).first() - upload.is_generated = generate_state - db.session.commit() - - check_map_size(upload_folder, user_currently_used_space, upload_uuid) - return generate_state - - -@celery.task(name='generate_geojson_file_upload') -def generate_geojson(upload_folder, layer_type, upload_uuid, user_currently_used_space): - ''' - This function is used to generate the geojson of a layer in the db. - :param upload_folder: the folder of the upload - :param layer_type: the name of the layer type choosen for the input - :param upload_uuid: the uuid of the upload - :param user_currently_used_space: the space currently used by the user - ''' - upload_csv = upload_folder + '/data.csv' - - try: - geojson_file_path = upload_folder + '/data.json' - with open(geojson_file_path, 'w') as geojson_file: - json.dump(csv_to_geojson(upload_csv, layer_type), geojson_file) - except: - generate_state = 10 - else: - generate_state = 0 - - # updating generate state of upload - upload = Uploads.query.filter_by(uuid=upload_uuid).first() - upload.is_generated = generate_state - db.session.commit() - check_map_size(upload_folder, user_currently_used_space, upload_uuid) - return generate_state - - -def check_map_size(upload_folder, user_currently_used_space, upload_uuid): - ''' - This method is used to check the size of the file - :param upload_folder: the folder where the upload is stored - :param user_currently_used_space: the space already used by the user - :param upload_uuid: the uuid of the upload - :return: - ''' - size = 0 - for dirpath, dirnames, filenames in os.walk(upload_folder): - for f in filenames: - fp = os.path.join(dirpath, f) - size += float(os.path.getsize(fp)) / 1000000 - # we need to check if there is enough disk space for the dataset - total_used_space = user_currently_used_space + size - upload = Uploads.query.filter_by(uuid=upload_uuid).first() - if total_used_space > constants.USER_DISC_SPACE_AVAILABLE: - db.session.delete(upload) - shutil.rmtree(upload_folder) - else: - upload.size = size - db.session.commit() - - -def generate_csv_string(result): - ''' - This method will generate the csv stringIO containing the result of a query without extra data - :param result: the sql result of a csv export - :return resultIO: the StringIO result formatted appropriately - ''' - columns_name = result.keys() - # if the selection is empty, we return only the columns names - if result.rowcount == 0: - df = DataFrame(columns=columns_name) - - else: - df = DataFrame(result.fetchall(), columns=columns_name) - - # remove geom columns - try: - df = df.drop(['geometry'], axis=1) - except: - pass - try: - df = df.drop(['geom'], axis=1) - except: - pass - - result_io = StringIO() - df.to_csv(result_io, index=False, quoting=csv.QUOTE_NONNUMERIC) - result_io.seek(0) - - return result_io - - -def find_property_column(style_sheet, headers): - ''' - This method will find the column that contains the property - :param style_sheet: the style sheet - :param headers: the available headers of our CSV - :return: the property column name - ''' - # create the xml tree - try: - root = ET.fromstring(style_sheet) - except: - raise RequestException('Can\'t parse SLD file') - ns = { - 'se': 'http://www.opengis.net/se', - 'ogc': 'http://www.opengis.net/ogc' - } - - # read rules - rules = root.findall(".//se:Rule", ns) - # read rules without 'se' prefix if previous did not work - if len(rules) == 0: - rules = root.findall(".//{http://www.opengis.net/sld}Rule") - # raise exception if rules is empty - if len(rules) == 0: - print("Can't read rules of SLD file.") - - # read filters - filters = rules[0].findall('./ogc:Filter/ogc:And', ns) - if len(filters) == 0: - filters = rules[0].findall('ogc:Filter', ns) - if len(filters) == 0: - print("Can't find any filter in SLD file.") - - for filter_element in filters: - - rule = filter_element.find('./ogc:PropertyIsGreaterThanOrEqualTo', ns) - if rule is None: - rule = filter_element.find('./ogc:PropertyIsGreaterThan', ns) - - if rule is None: - rule = filter_element.find('./ogc:PropertyIsEqualTo', ns) - - if rule is None: - rule = filter_element.find('./ogc:PropertyIsLessOrEqualTo', ns) - - if rule is None: - rule = filter_element.find('./ogc:PropertyIsLessThan', ns) - - if rule is None: - continue - - property_name = rule.find('./ogc:PropertyName', ns).text - - if property_name in headers: - return property_name - - return None - -def extract_query_string_parameters(url): - ''' - This method will extract all parameters from a url. - :param url: the url to extract the parameters from - :return: the dictionary of parameters - ''' - params = {} - try: - qs = urlparse(url).query - params = parse_qs(qs) - except: - pass - - - return params - - -def build_sld_size_formula(etree_size, operator=''): - ''' - This recursive method will build the formula to compute the size of the SLD graphic. - :param etree_size: the size XML tree element from SLD - :return: the string containing the formula - ''' - prefix = '{http://www.opengis.net/ogc}' - _operator = '' - - children = etree_size.getchildren() - formula = '' - values = [] - for child in children: - tag = child.tag.replace(prefix, '') - if tag == 'Literal': - values.append(child.text) - elif tag == 'PropertyName': - values.append(child.text) - else: - if tag == 'Add': - _operator = '+' - elif tag == 'Div': - _operator = '/' - elif tag == 'Mul': - _operator = '*' - elif tag == 'Sub': - _operator = '-' - values.append(build_sld_size_formula(child, _operator)) - - formula = '(' + operator.join(values) + ')' - - - return formula - - -def generate_rule_dictionary(style_sheet): - ''' - This method will generate a dictionnary of rule giving the stylesheet - :param style_sheet: the sld stylesheet - :return: the dictionnary of rules - ''' - # create the xml tree - try: - root = ET.fromstring(style_sheet) - except: - raise RequestException('Can\'t parse SLD file') - ns = { - 'se': 'http://www.opengis.net/se', - 'ogc': 'http://www.opengis.net/ogc' - } - ns_xlink = '{http://www.w3.org/1999/xlink}' - - # read rules - rules = root.findall(".//se:Rule", ns) - # read rules without 'se' prefix if previous did not work - if len(rules) == 0: - rules = root.findall(".//{http://www.opengis.net/sld}Rule") - # raise exception if rules is empty - if len(rules) == 0: - print("Can't read rules of SLD file.") - - # get the list of rules - rules_dictionary = {} - i = 0 - - for rule in rules: - filters = rule.findall('ogc:Filter/ogc:And', ns) - if len(filters) == 0: - filters = rule.findall('ogc:Filter', ns) - if len(filters) == 0: - print("Can't find any filter in SLD file.") - greater_type = None - lesser_type = None - equal_type = None - - for filter_type in filters: - - greater = filter_type.find('ogc:PropertyIsGreaterThanOrEqualTo', ns) - if greater is not None: - greater_type = GREATER_OR_EQUAL - else: - greater = filter_type.find('ogc:PropertyIsGreaterThan', ns) - if greater is not None: - greater_type = GREATER - - lesser = filter_type.find('ogc:PropertyIsLessThanOrEqualTo', ns) - if lesser is not None: - lesser_type = LESSER_OR_EQUAL - else: - lesser = filter_type.find('ogc:PropertyIsLessThan', ns) - if lesser is not None: - lesser_type = LESSER - - equal = filter_type.find('ogc:PropertyIsEqualTo', ns) - if equal is not None: - equal_type = EQUAL - - if greater is not None: - greater = float(greater.find('ogc:Literal', ns).text) - if lesser is not None: - lesser = float(lesser.find('ogc:Literal', ns).text) - if equal is not None: - try: - equal = float(equal.find('ogc:Literal', ns).text) - except ValueError: - equal = equal.find('ogc:Literal', ns).text - except TypeError: - equal = '' - - # identify symbology - graphic = rule.find('se:PointSymbolizer/se:Graphic/se:Mark/se:WellKnownName', ns) - external_graphic = None - - if graphic is not None: - mark_name = graphic.text - fill = rule.find('se:PointSymbolizer/se:Graphic/se:Mark/se:Fill/se:SvgParameter', ns).text - stroke = rule.find('se:PointSymbolizer/se:Graphic/se:Mark/se:Stroke/se:SvgParameter', ns).text - size = rule.find('se:PointSymbolizer/se:Graphic/se:Size', ns).text - else: - # TODO: handle points as charts if found - # otherwise return default style - - # defaults - mark_name = 'circle' - fill = '#0099ff' - stroke = '#ffffff' - size = '30' - - # extract graphic - xpath = 'se:PointSymbolizer/se:Graphic'.replace('se:', '{http://www.opengis.net/sld}') - graphic = rule.find(xpath) - if graphic is not None: - # extract external graphic - xpath = 'se:ExternalGraphic'.replace('se:', '{http://www.opengis.net/sld}') - _external_graphic = graphic.find('se:ExternalGraphic'.replace('se:', '{http://www.opengis.net/sld}')) - if _external_graphic is not None: - online_resource = _external_graphic.find('se:OnlineResource'.replace('se:', '{http://www.opengis.net/sld}'), ns) - xlink_resource = online_resource.attrib['{0}href'.format(ns_xlink)] if online_resource is not None else '' - chart_params = extract_query_string_parameters(xlink_resource) - chart_formulas = None - # extract chart formula - try: - # chd = chart data (geoserver SLD) - chd = chart_params['chd'][0] - chd_arr = chd.split(':') - if chd_arr[0] == 't': - # get raw formula from chd - chart_formula_raw = chd_arr[1].replace(' ', '') - # find all formulas used to compute chart (get array of formulas) - chart_formulas = re.findall(r'\$\{([^\}]*)\}', chart_formula_raw) - else: - print('chart data type not supported') - except: - print('error while parsing chart data') - pass - - try: - graphic_format = _external_graphic.find('se:Format'.replace('se:', '{http://www.opengis.net/sld}')).text - except: - graphic_format = '' - - # extract formula to compute size - _size = rule.find('se:PointSymbolizer/se:Graphic/se:Size'.replace('se:', '{http://www.opengis.net/sld}')) - if len(_size.getchildren()) > 0: - size_formula = build_sld_size_formula(_size) # retrieve formula from xml tree - - # join parameters - external_graphic = { - 'type': graphic_format, - 'params': chart_params, - 'formulas': chart_formulas, - 'size_formula': size_formula - } - - rules_dictionary[i] = { - 'greater_type': greater_type, - 'lesser_type': lesser_type, - 'equal_type': equal_type, - 'greater': greater, - 'lesser': lesser, - 'equal': equal, - 'mark_name': mark_name, - 'fill': fill, - 'stroke': stroke, - 'size': size - } - - if external_graphic is not None: - rules_dictionary[i]['external_graphic'] = external_graphic - - i += 1 - - return rules_dictionary - - -def find_rule(literal, rules_dictionary): - ''' - This method will find a specific rule in a rule dictionary - :param literal: the value we need to check - :param rules_dictionary: the dictionary of the rules used in the stylesheet - :return style: the style corresponding to the rule - ''' - for rule_id, rule_info in rules_dictionary.items(): - if rule_info['greater_type'] == GREATER_OR_EQUAL: - if not literal >= rule_info['greater']: - continue - elif rule_info['greater_type'] == GREATER: - if not literal > rule_info['greater']: - continue - - if rule_info['lesser_type'] == LESSER_OR_EQUAL: - if not literal <= rule_info['lesser']: - continue - elif rule_info['lesser_type'] == LESSER: - if not literal < rule_info['lesser']: - continue - - if rule_info['equal_type'] == EQUAL: - if not literal == rule_info['equal']: - continue - - # standard rule - style = { - "name": rule_info['mark_name'], - "fill": rule_info['fill'], - "stroke": rule_info['stroke'], - "size": rule_info['size'] - } - - # handle external graphic (charts) - if 'external_graphic' in rule_info.keys(): - style['external_graphic'] = rule_info['external_graphic'] - - - return style - - - return {} - - - -def csv_to_geojson(url, layer_type): - ''' - This method will convert the CSV to a geojson file - :param url: the URL of the CSV file - :param layer_type: the type of the layer of the CSV file - :return: the geojson - ''' - features = [] - srid = None - output_srid = '4326' - sld_file = helper.get_style_from_geoserver(layer_type) - rule_dictionary = generate_rule_dictionary(sld_file) - filtered_columns = [ - "year", - "month", - "day", - "weekday", - "season", - "hour_of_day", - "hour_of_year", - "date" - ] - - # parse file - with open(url, 'r', encoding="utf-8-sig") as csvfile: - reader = csv.DictReader(csvfile, delimiter=',') - property_column = find_property_column(sld_file, reader.fieldnames) - - for row in reader: - geom = None - properties = {} - srid = row['srid'] - - # read each column - for field in reader.fieldnames: - # remove filtered columns - if field in filtered_columns: - continue - - value = row[field] - - # get geometry and reproject (transform) - if field == 'geometry_wkt' or field == 'geometry' or field == 'geom': - try: - wkt = shapely_wkt.loads(value) - geometry = shapely_geom.mapping(wkt) - if srid != '4326': - project = partial( - pyproj.transform, - pyproj.Proj(init='epsg:{0}'.format(srid)), - pyproj.Proj(init='epsg:4326') - ) - geom = transform(project, shapely_geom.shape(geometry)) - else: - geom = geometry - except: - geom = None - else: - properties[field] = value - - # find property value in rules to retrieve style - try: - # prevent None or empty value - val = row[property_column] - if val == 'None' or len(val) == 0: - val = 0 - - # try to parse number - style = find_rule(float(val), rule_dictionary) - except ValueError: - # if type is not number - style = find_rule(row[property_column], rule_dictionary) - except TypeError: - # if type is not str or number - style = {} - - # handle external graphic (charts) - if 'external_graphic' in style.keys(): - external_graphic = style['external_graphic'] - eg_params = external_graphic.get('params', {}) - eg_formulas = external_graphic.get('formulas', {}) - eg_size_formula = external_graphic.get('size_formula', None) - - # compute data from SLD formulas & build style - data = {} - chart_options = {} - try: - colors = eg_params['chco'][0].split(',') - except: - colors = ['845ec2', 'd65db1', 'ff6f91', 'ff9671', 'ffc75f', 'f9f871', '0081cf', '00dbad', '96ee86', '008f7a'] - i = 0 - for f in eg_formulas: - # compute data - data_header = 'Data {}'.format(i) - used_headers = [] - for h in reader.fieldnames: - if h in f: - f = f.replace(h, row[h]) - used_headers.append(h) - try: - result = eval(f) - except: - result = 0.0 - - if len(used_headers) > 0: - data_header = used_headers[0] - - data[data_header] = result - - # chart style - chart_options[data_header] = { - 'fillColor': '#{}'.format(colors[i]), - 'color': '#ffffff' - #'minValue': 0, - #'maxValue': 20, - #'maxHeight': 20, - } - - i = i + 1 - - # chart type - # TODO handle other chart types - # default chart type = pie - chart_type = 'p' - try: - chart_type = eg_params['cht'][0] # cht = chart type - except: - pass - - if chart_type == 'p': - chart_type = 'pie' - - style['name'] = 'chart' - style['chart_type'] = chart_type - style['data'] = data - style['chartOptions'] = chart_options - style.pop('external_graphic', None) - - # compute size based on formula - try: - style['size'] = eval(eg_size_formula.replace(property_column, row[property_column])) - except: - # keep default 'size' if computation fails - pass - - features.append(Feature(geometry=geom, properties=properties, style=style)) - - crs = { - "type": "name", - "properties": { - "name": "EPSG:{0}".format(output_srid) - } - } - - - return FeatureCollection(features, crs=crs) - - -def calculate_total_space(uploads): - ''' - This method will calculate the amount of disc space taken by a list of uploads - :param uploads: - :return: the used disk space - ''' - used_size = float(0) - - # sum of every size - for upload in uploads: - used_size += float(upload.size) - - return used_size - - -def allowed_file(filename): - ''' - This method will check if the file is allowed - :param filename: - :return: - ''' - return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS +import csv +import json +import os +import re +import shutil +import uuid +import xml.etree.ElementTree as ET +from functools import partial +from io import StringIO +from urllib.parse import parse_qs, urlparse + +import pandas as pd +from pandas import DataFrame + +import app.helper +import pyproj +import requests +import shapely.geometry as shapely_geom +import shapely.wkt as shapely_wkt +from app import celery, model +from geojson import Feature, FeatureCollection +from shapely.ops import transform + +from .. import constants +from .. import dbGIS as db +from .. import helper +from ..decorators.exceptions import RequestException + +ALLOWED_EXTENSIONS = set(['tif', 'csv']) +GREATER_OR_EQUAL = 'greaterOrEqual' +GREATER = 'greater' +LESSER_OR_EQUAL = 'lesserOrEqual' +LESSER = 'lesser' +EQUAL = 'equal' + +class Uploads(db.Model): + ''' + This class will describe the model of a file uploaded by a user + ''' + __tablename__ = 'uploads' + __table_args__ = ( + {"schema": 'user'} + ) + + id = db.Column(db.Integer, primary_key=True) + uuid = db.Column(db.String(255)) + name = db.Column(db.String(255)) + layer = db.Column(db.String(255)) + layer_type = db.Column(db.String(255)) + size = db.Column(db.Numeric) + url = db.Column(db.String(255)) + is_generated = db.Column(db.Integer) + user_id = db.Column(db.Integer, db.ForeignKey('user.users.id')) + + +@celery.task(name='generate_tiles_file_upload') +def generate_tiles(upload_folder, grey_tif, layer_type, upload_uuid, user_currently_used_space): + ''' + This function is used to generate the various tiles of a layer in the db. + :param upload_folder: the folder of the upload + :param grey_tif: the url to the input file + :param layer_type: the type of the layer chosen for the input + :param upload_uuid: the uuid of the upload + :param user_currently_used_space: the space currently used by the user + + ''' + # we set up the directory for the tif + directory_for_tiles = upload_folder + '/tiles' + + tile_path = directory_for_tiles + access_rights = 0o755 + try: + os.mkdir(tile_path, access_rights) + except OSError: + print ("Creation of the directory %s failed" % tile_path) + else: + print ("Successfully created the directory %s" % tile_path) + + rgb_tif = upload_folder + '/rgba.tif' + if layer_type != 'custom': + helper.colorize(layer_type, grey_tif, rgb_tif) + else: + args_gdal = app.helper.commands_in_array("gdal_translate -of GTiff -expand rgba {} {} -co COMPRESS=DEFLATE ".format(grey_tif, rgb_tif)) + app.helper.run_command(args_gdal) + + try: + # commands launch to obtain the level of zooms + args_tiles = app.helper.commands_in_array("python3 app/helper/gdal2tiles.py -p 'mercator' -s 'EPSG:3035' -w 'leaflet' -r 'average' -z '4-11' {} {} ".format(rgb_tif, tile_path)) + app.helper.run_command(args_tiles) + + except : + generate_state = 10 + else: + generate_state = 0 + + # updating generate state of upload + upload = Uploads.query.filter_by(url=grey_tif).first() + upload.is_generated = generate_state + db.session.commit() + + check_map_size(upload_folder, user_currently_used_space, upload_uuid) + return generate_state + + +@celery.task(name='generate_geojson_file_upload') +def generate_geojson(upload_folder, layer_type, upload_uuid, user_currently_used_space): + ''' + This function is used to generate the geojson of a layer in the db. + :param upload_folder: the folder of the upload + :param layer_type: the name of the layer type choosen for the input + :param upload_uuid: the uuid of the upload + :param user_currently_used_space: the space currently used by the user + ''' + upload_csv = upload_folder + '/data.csv' + + try: + geojson_file_path = upload_folder + '/data.json' + with open(geojson_file_path, 'w') as geojson_file: + json.dump(csv_to_geojson(upload_csv, layer_type), geojson_file) + except: + generate_state = 10 + else: + generate_state = 0 + + # updating generate state of upload + upload = Uploads.query.filter_by(uuid=upload_uuid).first() + upload.is_generated = generate_state + db.session.commit() + check_map_size(upload_folder, user_currently_used_space, upload_uuid) + return generate_state + + +def check_map_size(upload_folder, user_currently_used_space, upload_uuid): + ''' + This method is used to check the size of the file + :param upload_folder: the folder where the upload is stored + :param user_currently_used_space: the space already used by the user + :param upload_uuid: the uuid of the upload + :return: + ''' + size = 0 + for dirpath, dirnames, filenames in os.walk(upload_folder): + for f in filenames: + fp = os.path.join(dirpath, f) + size += float(os.path.getsize(fp)) / 1000000 + # we need to check if there is enough disk space for the dataset + total_used_space = user_currently_used_space + size + upload = Uploads.query.filter_by(uuid=upload_uuid).first() + if total_used_space > constants.USER_DISC_SPACE_AVAILABLE: + db.session.delete(upload) + shutil.rmtree(upload_folder) + else: + upload.size = size + db.session.commit() + + +def generate_csv_string(result): + ''' + This method will generate the csv stringIO containing the result of a query without extra data + :param result: the sql result of a csv export + :return resultIO: the StringIO result formatted appropriately + ''' + columns_name = result.keys() + # if the selection is empty, we return only the columns names + if result.rowcount == 0: + df = DataFrame(columns=columns_name) + + else: + df = DataFrame(result.fetchall(), columns=columns_name) + + # remove geom columns + try: + df = df.drop(['geometry'], axis=1) + except: + pass + try: + df = df.drop(['geom'], axis=1) + except: + pass + + result_io = StringIO() + df.to_csv(result_io, index=False, quoting=csv.QUOTE_NONNUMERIC) + result_io.seek(0) + + return result_io + + +def find_property_column(style_sheet, headers): + ''' + This method will find the column that contains the property + :param style_sheet: the style sheet + :param headers: the available headers of our CSV + :return: the property column name + ''' + # create the xml tree + try: + root = ET.fromstring(style_sheet) + except: + raise RequestException('Can\'t parse SLD file') + ns = { + 'se': 'http://www.opengis.net/se', + 'ogc': 'http://www.opengis.net/ogc' + } + + # read rules + rules = root.findall(".//se:Rule", ns) + # read rules without 'se' prefix if previous did not work + if len(rules) == 0: + rules = root.findall(".//{http://www.opengis.net/sld}Rule") + # raise exception if rules is empty + if len(rules) == 0: + print("Can't read rules of SLD file.") + + # read filters + filters = rules[0].findall('./ogc:Filter/ogc:And', ns) + if len(filters) == 0: + filters = rules[0].findall('ogc:Filter', ns) + if len(filters) == 0: + print("Can't find any filter in SLD file.") + + for filter_element in filters: + + rule = filter_element.find('./ogc:PropertyIsGreaterThanOrEqualTo', ns) + if rule is None: + rule = filter_element.find('./ogc:PropertyIsGreaterThan', ns) + + if rule is None: + rule = filter_element.find('./ogc:PropertyIsEqualTo', ns) + + if rule is None: + rule = filter_element.find('./ogc:PropertyIsLessOrEqualTo', ns) + + if rule is None: + rule = filter_element.find('./ogc:PropertyIsLessThan', ns) + + if rule is None: + continue + + property_name = rule.find('./ogc:PropertyName', ns).text + + if property_name in headers: + return property_name + + return None + +def extract_query_string_parameters(url): + ''' + This method will extract all parameters from a url. + :param url: the url to extract the parameters from + :return: the dictionary of parameters + ''' + params = {} + try: + qs = urlparse(url).query + params = parse_qs(qs) + except: + pass + + + return params + + +def build_sld_size_formula(etree_size, operator=''): + ''' + This recursive method will build the formula to compute the size of the SLD graphic. + :param etree_size: the size XML tree element from SLD + :return: the string containing the formula + ''' + prefix = '{http://www.opengis.net/ogc}' + _operator = '' + + children = etree_size.getchildren() + formula = '' + values = [] + for child in children: + tag = child.tag.replace(prefix, '') + if tag == 'Literal': + values.append(child.text) + elif tag == 'PropertyName': + values.append(child.text) + else: + if tag == 'Add': + _operator = '+' + elif tag == 'Div': + _operator = '/' + elif tag == 'Mul': + _operator = '*' + elif tag == 'Sub': + _operator = '-' + values.append(build_sld_size_formula(child, _operator)) + + formula = '(' + operator.join(values) + ')' + + + return formula + + +def generate_rule_dictionary(style_sheet): + ''' + This method will generate a dictionnary of rule giving the stylesheet + :param style_sheet: the sld stylesheet + :return: the dictionnary of rules + ''' + # create the xml tree + try: + root = ET.fromstring(style_sheet) + except: + raise RequestException('Can\'t parse SLD file') + ns = { + 'se': 'http://www.opengis.net/se', + 'ogc': 'http://www.opengis.net/ogc' + } + ns_xlink = '{http://www.w3.org/1999/xlink}' + + # read rules + rules = root.findall(".//se:Rule", ns) + # read rules without 'se' prefix if previous did not work + if len(rules) == 0: + rules = root.findall(".//{http://www.opengis.net/sld}Rule") + # raise exception if rules is empty + if len(rules) == 0: + print("Can't read rules of SLD file.") + + # get the list of rules + rules_dictionary = {} + i = 0 + + for rule in rules: + filters = rule.findall('ogc:Filter/ogc:And', ns) + if len(filters) == 0: + filters = rule.findall('ogc:Filter', ns) + if len(filters) == 0: + print("Can't find any filter in SLD file.") + greater_type = None + lesser_type = None + equal_type = None + + for filter_type in filters: + + greater = filter_type.find('ogc:PropertyIsGreaterThanOrEqualTo', ns) + if greater is not None: + greater_type = GREATER_OR_EQUAL + else: + greater = filter_type.find('ogc:PropertyIsGreaterThan', ns) + if greater is not None: + greater_type = GREATER + + lesser = filter_type.find('ogc:PropertyIsLessThanOrEqualTo', ns) + if lesser is not None: + lesser_type = LESSER_OR_EQUAL + else: + lesser = filter_type.find('ogc:PropertyIsLessThan', ns) + if lesser is not None: + lesser_type = LESSER + + equal = filter_type.find('ogc:PropertyIsEqualTo', ns) + if equal is not None: + equal_type = EQUAL + + if greater is not None: + greater = float(greater.find('ogc:Literal', ns).text) + if lesser is not None: + lesser = float(lesser.find('ogc:Literal', ns).text) + if equal is not None: + try: + equal = float(equal.find('ogc:Literal', ns).text) + except ValueError: + equal = equal.find('ogc:Literal', ns).text + except TypeError: + equal = '' + + # identify symbology + graphic = rule.find('se:PointSymbolizer/se:Graphic/se:Mark/se:WellKnownName', ns) + external_graphic = None + + if graphic is not None: + mark_name = graphic.text + fill = rule.find('se:PointSymbolizer/se:Graphic/se:Mark/se:Fill/se:SvgParameter', ns).text + stroke = rule.find('se:PointSymbolizer/se:Graphic/se:Mark/se:Stroke/se:SvgParameter', ns).text + size = rule.find('se:PointSymbolizer/se:Graphic/se:Size', ns).text + else: + # TODO: handle points as charts if found + # otherwise return default style + + # defaults + mark_name = 'circle' + fill = '#0099ff' + stroke = '#ffffff' + size = '30' + + # extract graphic + xpath = 'se:PointSymbolizer/se:Graphic'.replace('se:', '{http://www.opengis.net/sld}') + graphic = rule.find(xpath) + if graphic is not None: + # extract external graphic + xpath = 'se:ExternalGraphic'.replace('se:', '{http://www.opengis.net/sld}') + _external_graphic = graphic.find('se:ExternalGraphic'.replace('se:', '{http://www.opengis.net/sld}')) + if _external_graphic is not None: + online_resource = _external_graphic.find('se:OnlineResource'.replace('se:', '{http://www.opengis.net/sld}'), ns) + xlink_resource = online_resource.attrib['{0}href'.format(ns_xlink)] if online_resource is not None else '' + chart_params = extract_query_string_parameters(xlink_resource) + chart_formulas = None + # extract chart formula + try: + # chd = chart data (geoserver SLD) + chd = chart_params['chd'][0] + chd_arr = chd.split(':') + if chd_arr[0] == 't': + # get raw formula from chd + chart_formula_raw = chd_arr[1].replace(' ', '') + # find all formulas used to compute chart (get array of formulas) + chart_formulas = re.findall(r'\$\{([^\}]*)\}', chart_formula_raw) + else: + print('chart data type not supported') + except: + print('error while parsing chart data') + pass + + try: + graphic_format = _external_graphic.find('se:Format'.replace('se:', '{http://www.opengis.net/sld}')).text + except: + graphic_format = '' + + # extract formula to compute size + _size = rule.find('se:PointSymbolizer/se:Graphic/se:Size'.replace('se:', '{http://www.opengis.net/sld}')) + if len(_size.getchildren()) > 0: + size_formula = build_sld_size_formula(_size) # retrieve formula from xml tree + + # join parameters + external_graphic = { + 'type': graphic_format, + 'params': chart_params, + 'formulas': chart_formulas, + 'size_formula': size_formula + } + + rules_dictionary[i] = { + 'greater_type': greater_type, + 'lesser_type': lesser_type, + 'equal_type': equal_type, + 'greater': greater, + 'lesser': lesser, + 'equal': equal, + 'mark_name': mark_name, + 'fill': fill, + 'stroke': stroke, + 'size': size + } + + if external_graphic is not None: + rules_dictionary[i]['external_graphic'] = external_graphic + + i += 1 + + return rules_dictionary + + +def find_rule(literal, rules_dictionary): + ''' + This method will find a specific rule in a rule dictionary + :param literal: the value we need to check + :param rules_dictionary: the dictionary of the rules used in the stylesheet + :return style: the style corresponding to the rule + ''' + for rule_id, rule_info in rules_dictionary.items(): + if rule_info['greater_type'] == GREATER_OR_EQUAL: + if not literal >= rule_info['greater']: + continue + elif rule_info['greater_type'] == GREATER: + if not literal > rule_info['greater']: + continue + + if rule_info['lesser_type'] == LESSER_OR_EQUAL: + if not literal <= rule_info['lesser']: + continue + elif rule_info['lesser_type'] == LESSER: + if not literal < rule_info['lesser']: + continue + + if rule_info['equal_type'] == EQUAL: + if not literal == rule_info['equal']: + continue + + # standard rule + style = { + "name": rule_info['mark_name'], + "fill": rule_info['fill'], + "stroke": rule_info['stroke'], + "size": rule_info['size'] + } + + # handle external graphic (charts) + if 'external_graphic' in rule_info.keys(): + style['external_graphic'] = rule_info['external_graphic'] + + + return style + + + return {} + + + +def csv_to_geojson(url, layer_type): + ''' + This method will convert the CSV to a geojson file + :param url: the URL of the CSV file + :param layer_type: the type of the layer of the CSV file + :return: the geojson + ''' + features = [] + srid = None + output_srid = '4326' + sld_file = helper.get_style_from_geoserver(layer_type) + rule_dictionary = generate_rule_dictionary(sld_file) + filtered_columns = [ + "year", + "month", + "day", + "weekday", + "season", + "hour_of_day", + "hour_of_year", + "date" + ] + + # parse file + with open(url, 'r', encoding="utf-8-sig") as csvfile: + reader = csv.DictReader(csvfile, delimiter=',') + property_column = find_property_column(sld_file, reader.fieldnames) + + for row in reader: + geom = None + properties = {} + srid = row['srid'] + + # read each column + for field in reader.fieldnames: + # remove filtered columns + if field in filtered_columns: + continue + + value = row[field] + + # get geometry and reproject (transform) + if field == 'geometry_wkt' or field == 'geometry' or field == 'geom': + try: + wkt = shapely_wkt.loads(value) + geometry = shapely_geom.mapping(wkt) + if srid != '4326': + project = partial( + pyproj.transform, + pyproj.Proj(init='epsg:{0}'.format(srid)), + pyproj.Proj(init='epsg:4326') + ) + geom = transform(project, shapely_geom.shape(geometry)) + else: + geom = geometry + except: + geom = None + else: + properties[field] = value + + # find property value in rules to retrieve style + try: + # prevent None or empty value + val = row[property_column] + if val == 'None' or len(val) == 0: + val = 0 + + # try to parse number + style = find_rule(float(val), rule_dictionary) + except ValueError: + # if type is not number + style = find_rule(row[property_column], rule_dictionary) + except TypeError: + # if type is not str or number + style = {} + + # handle external graphic (charts) + if 'external_graphic' in style.keys(): + external_graphic = style['external_graphic'] + eg_params = external_graphic.get('params', {}) + eg_formulas = external_graphic.get('formulas', {}) + eg_size_formula = external_graphic.get('size_formula', None) + + # compute data from SLD formulas & build style + data = {} + chart_options = {} + try: + colors = eg_params['chco'][0].split(',') + except: + colors = ['845ec2', 'd65db1', 'ff6f91', 'ff9671', 'ffc75f', 'f9f871', '0081cf', '00dbad', '96ee86', '008f7a'] + i = 0 + for f in eg_formulas: + # compute data + data_header = 'Data {}'.format(i) + used_headers = [] + for h in reader.fieldnames: + if h in f: + f = f.replace(h, row[h]) + used_headers.append(h) + try: + result = eval(f) + except: + result = 0.0 + + if len(used_headers) > 0: + data_header = used_headers[0] + + data[data_header] = result + + # chart style + chart_options[data_header] = { + 'fillColor': '#{}'.format(colors[i]), + 'color': '#ffffff' + #'minValue': 0, + #'maxValue': 20, + #'maxHeight': 20, + } + + i = i + 1 + + # chart type + # TODO handle other chart types + # default chart type = pie + chart_type = 'p' + try: + chart_type = eg_params['cht'][0] # cht = chart type + except: + pass + + if chart_type == 'p': + chart_type = 'pie' + + style['name'] = 'chart' + style['chart_type'] = chart_type + style['data'] = data + style['chartOptions'] = chart_options + style.pop('external_graphic', None) + + # compute size based on formula + try: + style['size'] = eval(eg_size_formula.replace(property_column, row[property_column])) + except: + # keep default 'size' if computation fails + pass + + features.append(Feature(geometry=geom, properties=properties, style=style)) + + crs = { + "type": "name", + "properties": { + "name": "EPSG:{0}".format(output_srid) + } + } + + + return FeatureCollection(features, crs=crs) + + +def calculate_total_space(uploads): + ''' + This method will calculate the amount of disc space taken by a list of uploads + :param uploads: + :return: the used disk space + ''' + used_size = float(0) + + # sum of every size + for upload in uploads: + used_size += float(upload.size) + + return used_size + + +def allowed_file(filename): + ''' + This method will check if the file is allowed + :param filename: + :return: + ''' + return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS diff --git a/api/app/models/user.py b/api/app/models/user.py index 52cd4731..fd87b42b 100644 --- a/api/app/models/user.py +++ b/api/app/models/user.py @@ -1,10 +1,13 @@ -from .. import dbGIS as db -from flask_security import UserMixin -from .role import Role -from itsdangerous import (TimedJSONWebSignatureSerializer, BadSignature, SignatureExpired) +import datetime import sys + +from flask_security import UserMixin +from itsdangerous import (BadSignature, SignatureExpired, + TimedJSONWebSignatureSerializer) + +from .. import dbGIS as db from ..constants import FLASK_SECRET_KEY -import datetime +from .role import Role class User(db.Model, UserMixin): diff --git a/api/app/models/wwtp.py b/api/app/models/wwtp.py index 5cfa703e..e359ce12 100644 --- a/api/app/models/wwtp.py +++ b/api/app/models/wwtp.py @@ -21,4 +21,3 @@ def __repr__(self): str_date = self.date.strftime("%Y-%m-%d") return "" % ( self.gid, str_date, self.capacity, self.power, self.unit, self.geom) - diff --git a/api/app/sql_queries.py b/api/app/sql_queries.py index becef0b6..976bc255 100644 --- a/api/app/sql_queries.py +++ b/api/app/sql_queries.py @@ -48,7 +48,6 @@ def vector_query_hectares(vector_table_requested, geometry,toCRS): "AND STAT_LEVL_ = 0 AND year = to_date('2013', 'YYYY') ) " + \ "select * from stat." + vector_table_requested + ",subAreas " + \ "where fk_nuts_gid = subAreas.gid" - print ('query ',query) return query @@ -61,10 +60,10 @@ def vector_query_nuts(vector_table_requested, area_selected): """ vector_table_requested = str(vector_table_requested) query= "with selected_zone as ( SELECT geom as geom from geo.nuts where nuts_id IN("+ area_selected+") " \ - "AND year = to_date('2013', 'YYYY') ), subAreas as ( SELECT distinct geo.nuts.nuts_id,geo.nuts.gid " \ - "FROM selected_zone, geo.nuts where ST_Intersects( geo.nuts.geom, selected_zone.geom ) " \ - "AND geo.nuts.STAT_LEVL_ = 0 AND geo.nuts.year = to_date('2013', 'YYYY') ) " \ - "select * from stat." + vector_table_requested + ",subAreas where fk_nuts_gid = subAreas.gid" + "AND year = to_date('2013', 'YYYY') ), subAreas as ( SELECT distinct geo.nuts.nuts_id,geo.nuts.gid " \ + "FROM selected_zone, geo.nuts where ST_Intersects( geo.nuts.geom, selected_zone.geom ) " \ + "AND geo.nuts.STAT_LEVL_ = 0 AND geo.nuts.year = to_date('2013', 'YYYY') ) " \ + "select * from stat." + vector_table_requested + ",subAreas where fk_nuts_gid = subAreas.gid" return query @@ -76,23 +75,20 @@ def vector_query_lau(vector_table_requested, area_selected,toCRS): :param geometry: :return """ - query= "with selected_zone as ( SELECT geom" \ - " from public.tbl_lau1_2 where comm_id IN("+ area_selected+") )," \ - " subAreas as ( SELECT distinct geo.nuts.nuts_id, geo.nuts.gid FROM selected_zone, geo.nuts " \ - "where ST_Intersects( geo.nuts.geom, selected_zone.geom ) AND geo.nuts.STAT_LEVL_ = 0 " \ - "AND geo.nuts.year = to_date('2013', 'YYYY') ) select * from stat." + vector_table_requested + ",subAreas where fk_nuts_gid = subAreas.gid" + " from public.tbl_lau1_2 where comm_id IN("+ area_selected+") )," \ + " subAreas as ( SELECT distinct geo.nuts.nuts_id, geo.nuts.gid FROM selected_zone, geo.nuts " \ + "where ST_Intersects( geo.nuts.geom, selected_zone.geom ) AND geo.nuts.STAT_LEVL_ = 0 " \ + "AND geo.nuts.year = to_date('2013', 'YYYY') ) select * from stat." + vector_table_requested + ",subAreas where fk_nuts_gid = subAreas.gid" return query - def nuts_within_the_selection(geometry,toCRS): query= "SELECT nuts.nuts_id FROM geo.nuts " + \ "where ST_Intersects( nuts.geom," + \ " "+transformGeo(geometry,toCRS)+" ) AND geo.nuts.STAT_LEVL_ = 2 AND year = to_date('2013', 'YYYY')" - print ('query ',query) - return query + def nuts2_within_the_selection_nuts_lau(scalevalue, geometry,toCRS): """ this function will return the need scale query select the scalevalue @@ -112,11 +108,11 @@ def nuts2_within_the_selection_nuts_lau(scalevalue, geometry,toCRS): return None def nuts2_within_the_selection_nuts(area_selected,toCRS): - query= "with selected_zone as ( SELECT geom as geom from geo.nuts where nuts_id IN("+ area_selected+") " \ - "AND year = to_date('2013', 'YYYY') ), subAreas as ( SELECT distinct geo.nuts.nuts_id,geo.nuts.gid " \ - "FROM selected_zone, geo.nuts where ST_Intersects( geo.nuts.geom, selected_zone.geom ) " \ - "AND geo.nuts.STAT_LEVL_ = 2 AND geo.nuts.year = to_date('2013', 'YYYY') ) " \ - "select subAreas.nuts_id from subAreas" + query = "with selected_zone as ( SELECT geom as geom from geo.nuts where nuts_id IN("+ area_selected+") " \ + "AND year = to_date('2013', 'YYYY') ), subAreas as ( SELECT distinct geo.nuts.nuts_id,geo.nuts.gid " \ + "FROM selected_zone, geo.nuts where ST_Intersects( geo.nuts.geom, selected_zone.geom ) " \ + "AND geo.nuts.STAT_LEVL_ = 2 AND geo.nuts.year = to_date('2013', 'YYYY') ) " \ + "select subAreas.nuts_id from subAreas" return query @@ -125,9 +121,7 @@ def nuts2_within_the_selection_nuts(area_selected,toCRS): def nuts2_within_the_selection_lau(area_selected,toCRS): query= "with selected_zone as ( SELECT geom" \ " from public.tbl_lau1_2 where comm_id IN("+ area_selected+") )," \ - " subAreas as ( SELECT distinct geo.nuts.nuts_id, geo.nuts.gid FROM selected_zone, geo.nuts " \ - "where ST_Intersects( geo.nuts.geom, selected_zone.geom ) AND geo.nuts.STAT_LEVL_ = 2 " \ - "AND geo.nuts.year = to_date('2013', 'YYYY') ) select subAreas.nuts_id from subAreas" - print ('query', query) + " subAreas as ( SELECT distinct geo.nuts.nuts_id, geo.nuts.gid FROM selected_zone, geo.nuts " \ + "where ST_Intersects( geo.nuts.geom, selected_zone.geom ) AND geo.nuts.STAT_LEVL_ = 2 " \ + "AND geo.nuts.year = to_date('2013', 'YYYY') ) select subAreas.nuts_id from subAreas" return query - diff --git a/api/celery_worker_local.py b/api/celery_worker_local.py index ed8baedc..ab00b901 100644 --- a/api/celery_worker_local.py +++ b/api/celery_worker_local.py @@ -1,9 +1,11 @@ #!/usr/bin/env python import os +from pathlib import Path + from app import celery, create_app from dotenv import load_dotenv -from pathlib import Path + env_path = Path('../.env') load_dotenv(dotenv_path=env_path) diff --git a/api/config/development.py b/api/config/development.py index 4fa3bea7..0fde6f25 100644 --- a/api/config/development.py +++ b/api/config/development.py @@ -1,4 +1,6 @@ -import os, sys, importlib.util +import importlib.util +import os +import sys constants_path = os.path.join(os.path.dirname(__file__), "..", "app", "constants.py") constants_spec = importlib.util.spec_from_file_location('constants', constants_path) diff --git a/api/config/production.py b/api/config/production.py index c59d2e5b..724c9439 100644 --- a/api/config/production.py +++ b/api/config/production.py @@ -1,4 +1,6 @@ -import os, sys, importlib.util +import importlib.util +import os +import sys constants_path = os.path.join(os.path.dirname(__file__), "..", "app", "constants.py") constants_spec = importlib.util.spec_from_file_location('constants', constants_path) diff --git a/api/consumer_cm_register.local.py b/api/consumer_cm_register.local.py index 968ae407..1dfcf244 100644 --- a/api/consumer_cm_register.local.py +++ b/api/consumer_cm_register.local.py @@ -1,14 +1,16 @@ -from dotenv import load_dotenv -from pathlib import Path -env_path = Path('../.env') -load_dotenv(dotenv_path=env_path) - import logging +import socket +from pathlib import Path + import pika +import requests from app import constants +from dotenv import load_dotenv from run import application -import socket -import requests + +env_path = Path('../.env') +load_dotenv(dotenv_path=env_path) + LOG_FORMAT = ('%(levelname) -10s %(asctime)s %(name) -30s %(funcName) ' diff --git a/api/consumer_cm_register.py b/api/consumer_cm_register.py index 43d2eb04..78f6d9e5 100644 --- a/api/consumer_cm_register.py +++ b/api/consumer_cm_register.py @@ -1,9 +1,11 @@ import logging +import socket + import pika +import requests from app import constants from run import application -import socket -import requests + LOG_FORMAT = ('%(levelname) -10s %(asctime)s %(name) -30s %(funcName) ' '-35s %(lineno) -5d: %(message)s') LOGGER = logging.getLogger(__name__) diff --git a/api/gunicorn-config_eeg.py b/api/gunicorn-config_eeg.py index 95c1161f..ea85ccac 100644 --- a/api/gunicorn-config_eeg.py +++ b/api/gunicorn-config_eeg.py @@ -2,4 +2,3 @@ bind = "0.0.0.0:5000" workers = 15 - diff --git a/api/producer_cm_alive.local.py b/api/producer_cm_alive.local.py index b04b24ef..50230487 100644 --- a/api/producer_cm_alive.local.py +++ b/api/producer_cm_alive.local.py @@ -2,14 +2,13 @@ #!/usr/bin/env python -import uuid -import time import logging -import pika +import time +import uuid -from app.model import getCMList, delete_cm +import pika from app import constants - +from app.model import delete_cm, getCMList LOG_FORMAT = ('%(levelname) -10s %(asctime)s %(name) -30s %(funcName) ' '-35s %(lineno) -5d: %(message)s') @@ -75,5 +74,3 @@ def call(self, queue_name): else: LOGGER.info("[HTAPI] is going to delete: %s ",str(cm_id)) delete_cm(str(cm_id)) - - diff --git a/api/producer_cm_alive.py b/api/producer_cm_alive.py index ffaaeeec..2ce2b883 100644 --- a/api/producer_cm_alive.py +++ b/api/producer_cm_alive.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- #!/usr/bin/env python -import uuid -import time import logging -import pika +import time +import uuid -from app.model import getCMList, delete_cm +import pika from app import constants - +from app.model import delete_cm, getCMList LOG_FORMAT = ('%(levelname) -10s %(asctime)s %(name) -30s %(funcName) ' '-35s %(lineno) -5d: %(message)s') @@ -73,5 +72,3 @@ def call(self, queue_name): else: LOGGER.info("[HTAPI] is going to delete: %s ",str(cm_id)) delete_cm(str(cm_id)) - - diff --git a/api/run.local.py b/api/run.local.py index 090786d8..ba4309df 100644 --- a/api/run.local.py +++ b/api/run.local.py @@ -1,12 +1,14 @@ import os +from pathlib import Path + +from app import create_app, log +from app.model import init_sqlite_caculation_module_database from dotenv import load_dotenv -from pathlib import Path + env_path = Path('../.env') load_dotenv(dotenv_path=env_path) -from app import create_app, log -from app.model import init_sqlite_caculation_module_database application = create_app(os.environ.get('ENVIRONMENT')) @@ -14,4 +16,3 @@ with application.app_context(): init_sqlite_caculation_module_database() application.run(host='0.0.0.0', threaded=True) - diff --git a/api/run.py b/api/run.py index 658b1663..045ee30c 100644 --- a/api/run.py +++ b/api/run.py @@ -9,4 +9,3 @@ with application.app_context(): init_sqlite_caculation_module_database() application.run(host='0.0.0.0', threaded=True) - diff --git a/api/run_all_local_terminal.sh b/api/run_all_local_terminal.sh index d87e2ec0..e0e8c5e8 100644 --- a/api/run_all_local_terminal.sh +++ b/api/run_all_local_terminal.sh @@ -1,9 +1,10 @@ -#!/usr/bin/env bash - -gnome-terminal -e "python producer_cm_alive.local.py" --title="HTAPI: producer_cm_alive" - -gnome-terminal -e "python run.local.py" --title="HTAPI: run API" - -gnome-terminal -e "python consumer_cm_register.local.py" --title="HTAPI: consumer_cm_register" - -gnome-terminal -e "celery -A celery_worker_local.celery worker --loglevel=info" --title="HTAPI: celery3" +#!/usr/bin/env bash +TERM=gnome-terminal + +$TERM -e "python producer_cm_alive.local.py" --title="HTAPI: producer_cm_alive" + +$TERM -e "python run.local.py" --title="HTAPI: run API" + +$TERM -e "python consumer_cm_register.local.py" --title="HTAPI: consumer_cm_register" + +$TERM -e "celery -A celery_worker_local.celery worker --loglevel=info" --title="HTAPI: celery3" diff --git a/api/tests.py b/api/tests.py index 7023cc67..02c1cdfc 100644 --- a/api/tests.py +++ b/api/tests.py @@ -1,10 +1,8 @@ import unittest + import coverage -from tests.routes import test_indicators from app.models.indicators import layersData -import coverage - - +from tests.routes import test_indicators if __name__ == "__main__": suite = unittest.TestLoader().loadTestsFromTestCase(test_indicators.TestIndicators) @@ -12,5 +10,3 @@ suite = unittest.TestSuite() suite.addTest(test_indicators.TestIndicators(layersData[layer],'test_tablenameschema_exists')) """ unittest.TextTestRunner(verbosity=2).run(suite) - - diff --git a/api/tests/routes/indicators/__init__.py b/api/tests/routes/indicators/__init__.py index 8c0957d9..760759e3 100644 --- a/api/tests/routes/indicators/__init__.py +++ b/api/tests/routes/indicators/__init__.py @@ -1 +1 @@ -from . import test_indicators \ No newline at end of file +from . import test_indicators diff --git a/api/tests/routes/indicators/test_indicators.py b/api/tests/routes/indicators/test_indicators.py index c000c8f8..e8457661 100644 --- a/api/tests/routes/indicators/test_indicators.py +++ b/api/tests/routes/indicators/test_indicators.py @@ -6,8 +6,8 @@ from app.models.indicators import layersData from app.sql_queries import get_exists_table_query -from .payloads import nuts3_stat from ..test_client import TestClient +from .payloads import nuts3_stat class TestIndicators(unittest.TestCase): @@ -107,4 +107,4 @@ def test_tablenameschema_exists(self): with self.subTest(tablename=layersData[layer]['tablename']+"_lau",schema_scalelvl=layersData[layer]['schema_scalelvl']): sql_query = get_exists_table_query(tbname=layersData[layer]['tablename']+"_lau", schema=layersData[layer]['schema_scalelvl']) query = query_geographic_database(sql_query).fetchone() - self.assertTrue(bool(query[0]) == True) \ No newline at end of file + self.assertTrue(bool(query[0]) == True) diff --git a/pytest_suit/routes/__init__.py b/pytest_suit/routes/__init__.py index f189cba7..d05dc22e 100644 --- a/pytest_suit/routes/__init__.py +++ b/pytest_suit/routes/__init__.py @@ -1,5 +1,7 @@ -from .. import BASE_URL import os + +from .. import BASE_URL + dirname = os.path.dirname(__file__) test_csv_file = os.path.join(dirname, 'test_assets/test.csv') diff --git a/pytest_suit/routes/snapshot/__init__.py b/pytest_suit/routes/snapshot/__init__.py index 9f303e94..7a956045 100644 --- a/pytest_suit/routes/snapshot/__init__.py +++ b/pytest_suit/routes/snapshot/__init__.py @@ -1,4 +1,4 @@ from .. import BASE_URL from ..user import test_token -test_config = 'This is my test save' \ No newline at end of file +test_config = 'This is my test save' diff --git a/pytest_suit/routes/snapshot/test_addveSnapshot.py b/pytest_suit/routes/snapshot/test_addveSnapshot.py index c3256f5a..faa06acc 100644 --- a/pytest_suit/routes/snapshot/test_addveSnapshot.py +++ b/pytest_suit/routes/snapshot/test_addveSnapshot.py @@ -1,7 +1,8 @@ +from unittest import TestCase + import requests -from unittest import TestCase -from . import BASE_URL, test_token, test_config +from . import BASE_URL, test_config, test_token url = BASE_URL + '/snapshot/add' diff --git a/pytest_suit/routes/snapshot/test_listSnapshot.py b/pytest_suit/routes/snapshot/test_listSnapshot.py index d75b22bb..73ce8948 100644 --- a/pytest_suit/routes/snapshot/test_listSnapshot.py +++ b/pytest_suit/routes/snapshot/test_listSnapshot.py @@ -1,8 +1,9 @@ -import requests - import unittest from unittest import TestCase -from . import BASE_URL, test_token, test_config + +import requests + +from . import BASE_URL, test_config, test_token url = BASE_URL + '/snapshot/list' diff --git a/pytest_suit/routes/snapshot/test_loadSnapshot.py b/pytest_suit/routes/snapshot/test_loadSnapshot.py index 3c3f4eff..1d71ab1b 100644 --- a/pytest_suit/routes/snapshot/test_loadSnapshot.py +++ b/pytest_suit/routes/snapshot/test_loadSnapshot.py @@ -1,8 +1,9 @@ -import requests - import unittest from unittest import TestCase -from . import BASE_URL, test_token, test_config + +import requests + +from . import BASE_URL, test_config, test_token url = BASE_URL + '/snapshot/load' diff --git a/pytest_suit/routes/snapshot/test_updateSnapshot.py b/pytest_suit/routes/snapshot/test_updateSnapshot.py index 7d53b200..1ff51082 100644 --- a/pytest_suit/routes/snapshot/test_updateSnapshot.py +++ b/pytest_suit/routes/snapshot/test_updateSnapshot.py @@ -1,7 +1,8 @@ +from unittest import TestCase + import requests -from unittest import TestCase -from . import BASE_URL, test_token, test_config +from . import BASE_URL, test_config, test_token url = BASE_URL + '/snapshot/update' @@ -76,4 +77,4 @@ def test_post_user_unidentified(self): expected_status = '539' - assert output.json()['error']['status'] == expected_status \ No newline at end of file + assert output.json()['error']['status'] == expected_status diff --git a/pytest_suit/routes/snapshot/test_zdeleteSnapshot.py b/pytest_suit/routes/snapshot/test_zdeleteSnapshot.py index 45112369..66078a7a 100644 --- a/pytest_suit/routes/snapshot/test_zdeleteSnapshot.py +++ b/pytest_suit/routes/snapshot/test_zdeleteSnapshot.py @@ -1,7 +1,8 @@ +from unittest import TestCase + import requests -from unittest import TestCase -from . import BASE_URL, test_token, test_config +from . import BASE_URL, test_config, test_token url = BASE_URL + '/snapshot/delete' @@ -72,4 +73,4 @@ def test_delete_user_unidentified(self): expected_status = '539' - assert output.json()['error']['status'] == expected_status \ No newline at end of file + assert output.json()['error']['status'] == expected_status diff --git a/pytest_suit/routes/uploads/__init__.py b/pytest_suit/routes/uploads/__init__.py index 7947717b..9841b425 100644 --- a/pytest_suit/routes/uploads/__init__.py +++ b/pytest_suit/routes/uploads/__init__.py @@ -1,7 +1,8 @@ +import os +import uuid + from .. import BASE_URL, test_csv_file, test_tif_file from ..user import test_token -import uuid -import os test_upload_name = 'pytest_upload_csv' test_export_cm_layer_uuid = '3ef057c3-b65a-448f-9718-d46c33b9ec3b' diff --git a/pytest_suit/routes/uploads/test_addUploads.py b/pytest_suit/routes/uploads/test_addUploads.py index d412153b..b485beb9 100644 --- a/pytest_suit/routes/uploads/test_addUploads.py +++ b/pytest_suit/routes/uploads/test_addUploads.py @@ -1,8 +1,9 @@ +from unittest import TestCase + import requests -from unittest import TestCase -from . import BASE_URL, test_token, test_csv_file, test_upload_name from ..user.test_profileUser import test_first_name +from . import BASE_URL, test_csv_file, test_token, test_upload_name url = BASE_URL + '/upload/add' @@ -44,5 +45,3 @@ def test_post_user_unidentified(self): expected_status = '539' assert output.json()['error']['status'] == expected_status - - diff --git a/pytest_suit/routes/uploads/test_exportCMLayer.py b/pytest_suit/routes/uploads/test_exportCMLayer.py index f6939057..a5d992ae 100644 --- a/pytest_suit/routes/uploads/test_exportCMLayer.py +++ b/pytest_suit/routes/uploads/test_exportCMLayer.py @@ -1,9 +1,9 @@ +import os from unittest import TestCase import requests from . import BASE_URL, test_export_cm_layer_uuid, test_tif_file -import os url = BASE_URL + "/upload/export/cmLayer" diff --git a/pytest_suit/routes/uploads/test_exportCsvLau.py b/pytest_suit/routes/uploads/test_exportCsvLau.py index b828b7ab..15a27e07 100644 --- a/pytest_suit/routes/uploads/test_exportCsvLau.py +++ b/pytest_suit/routes/uploads/test_exportCsvLau.py @@ -58,4 +58,4 @@ def test_port_wrong_parameters(self): # expected_status = '532' - # assert output.json()['error']['status'] == expected_status \ No newline at end of file + # assert output.json()['error']['status'] == expected_status diff --git a/pytest_suit/routes/uploads/test_exportCsvNuts.py b/pytest_suit/routes/uploads/test_exportCsvNuts.py index 6da68e41..3a6d39a6 100644 --- a/pytest_suit/routes/uploads/test_exportCsvNuts.py +++ b/pytest_suit/routes/uploads/test_exportCsvNuts.py @@ -58,4 +58,4 @@ def test_port_wrong_parameters(self): # expected_status = '532' - # assert output.json()['error']['status'] == expected_status \ No newline at end of file + # assert output.json()['error']['status'] == expected_status diff --git a/pytest_suit/routes/uploads/test_listUploads.py b/pytest_suit/routes/uploads/test_listUploads.py index 0961bfcb..bd92722f 100644 --- a/pytest_suit/routes/uploads/test_listUploads.py +++ b/pytest_suit/routes/uploads/test_listUploads.py @@ -1,6 +1,7 @@ +from unittest import TestCase + import requests -from unittest import TestCase from . import BASE_URL, test_token from .test_addUploads import test_upload_name diff --git a/pytest_suit/routes/uploads/test_spaceUsedUploads.py b/pytest_suit/routes/uploads/test_spaceUsedUploads.py index 0e5f1a39..6a5a20d4 100644 --- a/pytest_suit/routes/uploads/test_spaceUsedUploads.py +++ b/pytest_suit/routes/uploads/test_spaceUsedUploads.py @@ -1,6 +1,7 @@ +from unittest import TestCase + import requests -from unittest import TestCase from . import BASE_URL, test_token url = BASE_URL + "/users/space_used" @@ -48,4 +49,4 @@ def test_post_user_unidentified(self): expected_status = '539' - assert output.json()['error']['status'] == expected_status \ No newline at end of file + assert output.json()['error']['status'] == expected_status diff --git a/pytest_suit/routes/uploads/test_zdownloadUploads.py b/pytest_suit/routes/uploads/test_zdownloadUploads.py index 07266713..5ba2ba83 100644 --- a/pytest_suit/routes/uploads/test_zdownloadUploads.py +++ b/pytest_suit/routes/uploads/test_zdownloadUploads.py @@ -76,4 +76,3 @@ def test_download_upload_not_existing(self): expected_status = '543' assert output.json()['error']['status'] == expected_status - diff --git a/pytest_suit/routes/uploads/test_zzdeleteUploads.py b/pytest_suit/routes/uploads/test_zzdeleteUploads.py index d048d77d..41f71334 100644 --- a/pytest_suit/routes/uploads/test_zzdeleteUploads.py +++ b/pytest_suit/routes/uploads/test_zzdeleteUploads.py @@ -1,7 +1,8 @@ +from unittest import TestCase + import requests -from unittest import TestCase -from . import test_token, BASE_URL +from . import BASE_URL, test_token url = BASE_URL + "/upload/delete" @@ -75,4 +76,3 @@ def test_delete_upload_not_existing(self): expected_status = '543' assert output.json()['error']['status'] == expected_status - diff --git a/pytest_suit/routes/user/__init__.py b/pytest_suit/routes/user/__init__.py index 54386ba9..b7515d22 100644 --- a/pytest_suit/routes/user/__init__.py +++ b/pytest_suit/routes/user/__init__.py @@ -1,4 +1,5 @@ import requests + from .. import BASE_URL url = BASE_URL + "/users/login" diff --git a/pytest_suit/routes/user/test_activateUser.py b/pytest_suit/routes/user/test_activateUser.py index c614c962..bf03dc21 100644 --- a/pytest_suit/routes/user/test_activateUser.py +++ b/pytest_suit/routes/user/test_activateUser.py @@ -1,7 +1,9 @@ from unittest import TestCase -from .. import BASE_URL + import requests +from .. import BASE_URL + class TestActivateUser(TestCase): # The working method test has been removed, because the user is not deleted each time and deleting a user is diff --git a/pytest_suit/routes/user/test_getUserInformation.py b/pytest_suit/routes/user/test_getUserInformation.py index 434a1abe..e8c12c9f 100644 --- a/pytest_suit/routes/user/test_getUserInformation.py +++ b/pytest_suit/routes/user/test_getUserInformation.py @@ -1,8 +1,9 @@ +from unittest import TestCase + import requests -from unittest import TestCase -from . import test_token from .. import BASE_URL +from . import test_token class TestGetUserInformation(TestCase): @@ -53,4 +54,3 @@ def test_post_user_unidentified(self): expected_status= '539' assert output.json()['error']['status'] == expected_status - diff --git a/pytest_suit/routes/user/test_profileUser.py b/pytest_suit/routes/user/test_profileUser.py index 150fbad3..8df71716 100644 --- a/pytest_suit/routes/user/test_profileUser.py +++ b/pytest_suit/routes/user/test_profileUser.py @@ -1,8 +1,9 @@ +from unittest import TestCase + import requests -from unittest import TestCase -from . import test_token from .. import BASE_URL +from . import test_token test_last_name = 'toto' test_first_name = 'tata' @@ -59,4 +60,3 @@ def test_post_user_unidentified(self): expected_status = '539' assert output.json()['error']['status'] == expected_status - diff --git a/pytest_suit/routes/user/test_recoverPassword.py b/pytest_suit/routes/user/test_recoverPassword.py index 96d749d1..ec3c029b 100644 --- a/pytest_suit/routes/user/test_recoverPassword.py +++ b/pytest_suit/routes/user/test_recoverPassword.py @@ -1,5 +1,7 @@ -import requests from unittest import TestCase + +import requests + from .. import BASE_URL @@ -39,4 +41,4 @@ def test_post_wrong_token(self): expected_output = '536' error_status = output.json()['error']['status'] - assert error_status == expected_output \ No newline at end of file + assert error_status == expected_output diff --git a/pytest_suit/routes/user/test_userRegistering.py b/pytest_suit/routes/user/test_userRegistering.py index f478e0f4..03bbb209 100644 --- a/pytest_suit/routes/user/test_userRegistering.py +++ b/pytest_suit/routes/user/test_userRegistering.py @@ -1,7 +1,9 @@ from unittest import TestCase -from .. import BASE_URL + import requests +from .. import BASE_URL + class TestUserRegistering(TestCase): # The working method test has been removed, because the user is not deleted each time and deleting a user is diff --git a/pytest_suit/routes/user/test_zlogoutUser.py b/pytest_suit/routes/user/test_zlogoutUser.py index 17ff63a0..9fed99fc 100644 --- a/pytest_suit/routes/user/test_zlogoutUser.py +++ b/pytest_suit/routes/user/test_zlogoutUser.py @@ -1,7 +1,9 @@ -import requests from unittest import TestCase -from . import test_token + +import requests + from .. import BASE_URL +from . import test_token # the z in the name is to run this test last to logout (but before the 2nd login that will change the valid token diff --git a/pytest_suit/routes/user/test_zzaskingPasswordRecovery.py b/pytest_suit/routes/user/test_zzaskingPasswordRecovery.py index 8d769098..95c8c30c 100644 --- a/pytest_suit/routes/user/test_zzaskingPasswordRecovery.py +++ b/pytest_suit/routes/user/test_zzaskingPasswordRecovery.py @@ -1,7 +1,9 @@ from unittest import TestCase -from .. import BASE_URL + import requests +from .. import BASE_URL + class TestAskingPasswordRecovery(TestCase): def test_post_working(self): diff --git a/pytest_suit/routes/user/test_zzloginUser.py b/pytest_suit/routes/user/test_zzloginUser.py index 7b8a699a..8f8d9bce 100644 --- a/pytest_suit/routes/user/test_zzloginUser.py +++ b/pytest_suit/routes/user/test_zzloginUser.py @@ -1,5 +1,7 @@ -import requests from unittest import TestCase + +import requests + from .. import BASE_URL