From 7548153e6e21a46a93f321b1ee60ab6f397d91a5 Mon Sep 17 00:00:00 2001 From: Olivier Date: Tue, 30 Jun 2026 22:36:57 +0200 Subject: [PATCH 1/4] Update Makefile --- backend/Makefile | 82 ++++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/backend/Makefile b/backend/Makefile index c4cc6e4b..72b72315 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -1,55 +1,55 @@ catalog: # All 4 Trees - Inventaire forestier - coordo load kobotoolbox --action add data/all4trees/inventaire_for/20260519_InventaireForestier_QuestionnaireK.xlsx data/all4trees/inventaire_for/20260422_InventaireForestier_DonneesK.xlsx --package catalog/inventaire_for - coordo load file --action add data/all4trees/inventaire_for/20260422_InventaireForestier_DonneesExternes.xlsx --package catalog/inventaire_for - - coordo add-foreignkey adu.decay for_dw.decay --package catalog/inventaire_for - coordo add-foreignkey inv_for.proj for_samp.proj --package catalog/inventaire_for - coordo add-foreignkey inv_for.typ for_pop.typ --package catalog/inventaire_for - coordo add-foreignkey inv_for.loc2 for_weath.loc2 --package catalog/inventaire_for - coordo add-foreignkey inv_for._id for_soil._index --package catalog/inventaire_for - coordo add-foreignkey adu.adu_sp for_sp.adu_sp --package catalog/inventaire_for - coordo add-foreignkey tsbf_001.tsbf_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for - coordo add-foreignkey barba_001.barbA_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for - coordo add-foreignkey barbb_001.barbB_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for - coordo add-foreignkey barbc_001.barbC_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for - coordo add-foreignkey barbd_001.barbD_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for - coordo add-foreignkey tsbf_001.tsbf_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for - coordo add-foreignkey barba_001.barbA_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for - coordo add-foreignkey barbb_001.barbB_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for - coordo add-foreignkey barbc_001.barbC_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for - coordo add-foreignkey barbd_001.barbD_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for - coordo add-foreignkey tsbf_001.tsbf_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for - coordo add-foreignkey barba_001.barbA_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for - coordo add-foreignkey barbb_001.barbB_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for - coordo add-foreignkey barbc_001.barbC_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for - coordo add-foreignkey barbd_001.barbD_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for + coordo add kobotoolbox data/all4trees/inventaire_for/20260519_InventaireForestier_QuestionnaireK.xlsx data/all4trees/inventaire_for/20260422_InventaireForestier_DonneesK.xlsx --package catalog/inventaire_for + coordo add file data/all4trees/inventaire_for/20260422_InventaireForestier_DonneesExternes.xlsx --package catalog/inventaire_for + + coordo add foreignkey adu.decay for_dw.decay --package catalog/inventaire_for + coordo add foreignkey inv_for.proj for_samp.proj --package catalog/inventaire_for + coordo add foreignkey inv_for.typ for_pop.typ --package catalog/inventaire_for + coordo add foreignkey inv_for.loc2 for_weath.loc2 --package catalog/inventaire_for + coordo add foreignkey inv_for._id for_soil._index --package catalog/inventaire_for + coordo add foreignkey adu.adu_sp for_sp.adu_sp --package catalog/inventaire_for + coordo add foreignkey tsbf_001.tsbf_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for + coordo add foreignkey barba_001.barbA_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for + coordo add foreignkey barbb_001.barbB_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for + coordo add foreignkey barbc_001.barbC_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for + coordo add foreignkey barbd_001.barbD_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for + coordo add foreignkey tsbf_001.tsbf_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for + coordo add foreignkey barba_001.barbA_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for + coordo add foreignkey barbb_001.barbB_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for + coordo add foreignkey barbc_001.barbC_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for + coordo add foreignkey barbd_001.barbD_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for + coordo add foreignkey tsbf_001.tsbf_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for + coordo add foreignkey barba_001.barbA_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for + coordo add foreignkey barbb_001.barbB_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for + coordo add foreignkey barbc_001.barbC_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for + coordo add foreignkey barbd_001.barbD_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for # All 4 Trees - Inventaire Biologique - coordo load kobotoolbox --action add data/all4trees/inventaire_bio/20260519_InventaireBiologique_QuestionnaireK.xlsx data/all4trees/inventaire_bio/20260422_InventaireBiologique_DonneesK.xlsx --package catalog/inventaire_bio - coordo load file --action add data/all4trees/inventaire_bio/20260422_InventaireBiologique_DonneesExternes.xlsx --package catalog/inventaire_bio + coordo add kobotoolbox data/all4trees/inventaire_bio/20260519_InventaireBiologique_QuestionnaireK.xlsx data/all4trees/inventaire_bio/20260422_InventaireBiologique_DonneesK.xlsx --package catalog/inventaire_bio + coordo add file data/all4trees/inventaire_bio/20260422_InventaireBiologique_DonneesExternes.xlsx --package catalog/inventaire_bio - coordo add-foreignkey inv_bio.proj bio_samp.proj --package catalog/inventaire_bio - coordo add-foreignkey inv_bio.year bio_pop.year --package catalog/inventaire_bio - coordo add-foreignkey inv_001.tax3 bio_sp.tax3 --package catalog/inventaire_bio + coordo add foreignkey inv_bio.proj bio_samp.proj --package catalog/inventaire_bio + coordo add foreignkey inv_bio.year bio_pop.year --package catalog/inventaire_bio + coordo add foreignkey inv_001.tax3 bio_sp.tax3 --package catalog/inventaire_bio # All 4 Trees - Enquête ménage - coordo load kobotoolbox --action add data/all4trees/enquete/20260519_EnqueteMenage_QuestionnaireK.xlsx data/all4trees/enquete/20260422_EnqueteMenage_DonneesK.csv --package catalog/enquete - coordo load file --action add data/all4trees/enquete/20260422_EnqueteMenage_DonneesExternes.xlsx --package catalog/enquete + coordo add kobotoolbox data/all4trees/enquete/20260519_EnqueteMenage_QuestionnaireK.xlsx data/all4trees/enquete/20260422_EnqueteMenage_DonneesK.csv --package catalog/enquete + coordo add file data/all4trees/enquete/20260422_EnqueteMenage_DonneesExternes.xlsx --package catalog/enquete - coordo add-foreignkey enquete_menage.proj enq_samp.proj --package catalog/enquete - coordo add-foreignkey enquete_menage.typ enq_pop.typ --package catalog/enquete - coordo add-foreignkey enquete_menage.loc2 enq_gps.loc2 --package catalog/enquete - coordo add-foreignkey enquete_menage.fw_cod enq_fw.fw_cod --package catalog/enquete - coordo add-foreignkey enquete_menage.coal_unit enq_coal.coal_unit --package catalog/enquete + coordo add foreignkey enquete_menage.proj enq_samp.proj --package catalog/enquete + coordo add foreignkey enquete_menage.typ enq_pop.typ --package catalog/enquete + coordo add foreignkey enquete_menage.loc2 enq_gps.loc2 --package catalog/enquete + coordo add foreignkey enquete_menage.fw_cod enq_fw.fw_cod --package catalog/enquete + coordo add foreignkey enquete_menage.coal_unit enq_coal.coal_unit --package catalog/enquete # Seed - Survey - coordo load file --action add data/seed/survey.zip --package catalog/seed - coordo load file --action add data/seed/suivi_sites_reboises.zip --package catalog/seed - coordo load file --action add data/seed/ident_sites.zip --package catalog/seed - coordo load file --action add data/seed/reboisement_id.zip --package catalog/seed - coordo load file --action add data/seed/sensib_communautaire.zip --package catalog/seed + coordo add file data/seed/survey.zip --package catalog/seed + coordo add file data/seed/suivi_sites_reboises.zip --package catalog/seed + coordo add file data/seed/ident_sites.zip --package catalog/seed + coordo add file data/seed/reboisement_id.zip --package catalog/seed + coordo add file data/seed/sensib_communautaire.zip --package catalog/seed test: python manage.py test \ No newline at end of file From f2159a6257825ace0e6193db84a8b466ed5ddb28 Mon Sep 17 00:00:00 2001 From: Olivier Date: Sun, 5 Jul 2026 13:42:16 +0200 Subject: [PATCH 2/4] add endpoints and corresponding views to intereact with resources and data in a datapackage --- backend/maps/urls.py | 5 +- backend/maps/user_data.py | 134 ++++++++++++++++++++++++++++++++++++++ backend/maps/views.py | 99 ++++++++++++---------------- 3 files changed, 179 insertions(+), 59 deletions(-) create mode 100644 backend/maps/user_data.py diff --git a/backend/maps/urls.py b/backend/maps/urls.py index 0716d125..89b14e44 100644 --- a/backend/maps/urls.py +++ b/backend/maps/urls.py @@ -3,7 +3,10 @@ from . import views urlpatterns = [ - path("add-data/", views.add_data_view, name="maps-add-data"), + path("add-data/", views.add_resource_using_file_view, name="maps-add-data"), + path("remove-data/", views.remove_resource_using_file_view, name="maps-remove-data"), + path("append-data/", views.append_file_data_to_datapackage_view, name="maps-append-data"), + path("replace-data/", views.replace_datapackage_data_from_file_view, name="maps-replace-data"), re_path(r"^(?!dashboard)(?P\w+)", views.my_map_view, name="maps-data"), path("dashboard/", views.dashboard_view, name="dashboard-data"), ] diff --git a/backend/maps/user_data.py b/backend/maps/user_data.py new file mode 100644 index 00000000..34d11485 --- /dev/null +++ b/backend/maps/user_data.py @@ -0,0 +1,134 @@ +from tempfile import gettempdir +from pathlib import Path + +import chardet + +from django.http import JsonResponse +from coordo.loaders import Loader, get_file_loader + + + +def add_resource(request): + return modify_resource(request, "remove", "Resource successfully added to datapackage") + + +def remove_resource(request): + return modify_resource(request, "remove", "Resource successfully removed from datapackage") + + +def modify_resource(request, action: str, response_message: str): + """ + View for modifying (adding / removing) a DataPackage resource using an uplodaed file. + Expects a POST request with a body containing the 'file' and 'package' fields. + An optional field 'options' can also be provided. + The uploaded file is saved temporarily in the system's temporary folder, processed and removed at the end. + """ + file = get_data_file(request) + try: + try: + package = Path(request.POST["package"]) + options: dict = request.POST.get("options", {}) + except KeyError: + return JsonResponse( + {"error": "Invalid request format. The request must contain at least the 'package' field."}, + status=400 + ) + + try: + file_loader = get_file_loader(package, file, **options) + if action == "add": + file_loader.add() + elif action == "remove": + file_loader.remove() + except Exception as e: + return JsonResponse({'error': str(e)}, status=500) + + finally: + # in any case, delete the temporary file + file.unlink(missing_ok=True) + + return JsonResponse({ + 'message': response_message, + 'filename': file.name, + 'package': package + }) + + +def append_data(request): + return update_data(request, "append", "File successfully appended to datapackage resource data") + + +def replace_data(request): + return update_data(request, "replace", "File successfully replaced datapackage resource data") + + +def update_data(request, action: str, response_message: str): + """ + View for updating data from a file to a resource of a DataPackage. + Expects a POST request with a body containing the 'file', 'package' and 'action' fields + Another optional body field, named 'resource', allows specifying the target resource in case it is different from the file stem. + The uploaded file is saved temporarily in the system's temporary folder, processed and removed at the end. + """ + file = get_data_file(request) + try: + + try: + package = Path(request.POST["package"]) + options: dict = request.POST.get("options", {}) + target_resource_name: str = request.POST.get("resource", None) + except KeyError: + return JsonResponse( + {"error": "Invalid request format. The request must contain at least the 'package' field."}, + status=400 + ) + + try: + file_folder = get_file_loader(package, file, **options) + # find method of the file_folder with name 'method_name' + # and execute it with the appropriate arguments + if action == "append": + file_folder.append(target_resource_name) + elif action == "replace": + file_folder.replace(target_resource_name) + else: + raise ValueError(f"Invalid action name: {action}. It must be one of: 'append' or 'replace'") + except Exception as e: + return JsonResponse({'error': str(e)}, status=500) + + finally: + # in any case, delete the temporary file + file.unlink(missing_ok=True) + + return JsonResponse({ + 'message': response_message, + 'filename': file.name, + 'package': package + }) + + +def get_data_file(request) -> Path: + """ + Gets the uplodaed file from the request object. + Decode and re-encode the file as utf-8 to avoid downstream issues. + Writes the file to a temporary file in the system's temporary folder. + """ + if 'file' not in request.FILES: + return JsonResponse({'error': 'No file provided'}, status=400) + + uploaded_file = request.FILES['file'] + temp_file = Path(gettempdir()) / uploaded_file.name + + # get file content + file_content = uploaded_file.read() + + # detect the encoding using chardet, decode the content and re-encode as UTF-8 + encoding_info = chardet.detect(file_content) + detected_encoding = encoding_info['encoding'] + decoded_content = file_content.decode(detected_encoding) + utf8_content = decoded_content.encode('utf-8') + + # save the file temporarily + with open(temp_file, 'wb') as f: + f.write(utf8_content) + + return temp_file \ No newline at end of file diff --git a/backend/maps/views.py b/backend/maps/views.py index 00111f3a..6d46140b 100644 --- a/backend/maps/views.py +++ b/backend/maps/views.py @@ -1,16 +1,12 @@ -from pathlib import Path -from tempfile import gettempdir - -from coordo.loaders import FileLoader, ResourceAction from coordo.map import Map from django.conf import settings from django.http import JsonResponse, HttpResponseBadRequest from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt from django.contrib.auth.decorators import permission_required -import chardet -from . import stats + +from . import stats, user_data config_path = settings.BASE_DIR / "configs" / "config.json" map = Map.from_file(config_path) @@ -39,60 +35,47 @@ def dashboard_view(request, layer_id): return HttpResponseBadRequest(f'Layer "{layer_id}" not yet supported', status=501) + @require_POST @permission_required("users.add_data") -def add_data_view(request): +def add_resource_using_file_view(request): """ - View for adding a file to a DataPackage. - Expects a POST request with a body containing the 'file', 'package' and 'action' fields. - The uploaded file is saved temporarily in the system's temporary folder, processed and removed at the end. + View for adding a file (creating the corresponding resource(s)) to a DataPackage. + Expects a POST request with a body containing the 'file', 'package' fields. + The body can also contain a 'options' field. """ - if 'file' not in request.FILES: - return JsonResponse({'error': 'No file provided'}, status=400) - - uploaded_file = request.FILES['file'] - temp_file = Path(gettempdir()) / uploaded_file.name - - # get file content - file_content = uploaded_file.read() - - # detect the encoding using chardet, decode the content and re-encode as UTF-8 - encoding_info = chardet.detect(file_content) - detected_encoding = encoding_info['encoding'] - decoded_content = file_content.decode(detected_encoding) - utf8_content = decoded_content.encode('utf-8') - - try: - # save the file temporarily - with open(temp_file, 'wb') as f: - f.write(utf8_content) - - try: - package = Path(request.POST["package"]) - action = request.POST["action"] - except KeyError: - return JsonResponse( - {"error": "Invalid request format. The request must contain the 'package' and 'action' fields."}, - status=400 - ) - - if action not in ResourceAction: - return JsonResponse( - {"error": "Invalid action. The action must be one of: " + ", ".join(ResourceAction)}, - status=400 - ) - - try: - FileLoader(package, temp_file, action).etl() - except Exception as e: - return JsonResponse({'error': str(e)}, status=500) - - finally: - # in any case, delete the temporary file - temp_file.unlink(missing_ok=True) - - return JsonResponse({ - 'message': 'File uploaded successfully', - 'filename': uploaded_file.name - }) + return user_data.add_resource(request) + + +@require_POST +@permission_required("users.add_data") +def remove_resource_using_file_view(request): + """ + View for removing resource(s) cooresponding a file to a DataPackage. + Expects a POST request with a body containing the 'file', 'package' fields. + """ + return user_data.remove_resource(request) + + +@require_POST +@permission_required("users.add_data") +def append_file_data_to_datapackage_view(request): + """ + View for appending the data contained in a file to the corresponding resources of a DataPackage. + Expects a POST request with a body containing at least the 'file', 'package' fields. + Optionally, a target resource name can be provided as a 'resource' field. + """ + return user_data.append_data(request) + + +@require_POST +@permission_required("users.add_data") +def replace_datapackage_data_from_file_view(request): + """ + View for removing the data contained in a file from the corresponding resources of a DataPackage. + Expects a POST request with a body containing at least the 'file', 'package' fields. + Optionally, a target resource name can be provided as a 'resource' field. + """ + return user_data.replace_data(request) + \ No newline at end of file From 25a8dff01b0d67d0b5d0f7159e59a33baf766940 Mon Sep 17 00:00:00 2001 From: Olivier Date: Sun, 5 Jul 2026 22:48:53 +0200 Subject: [PATCH 3/4] add tests --- backend/maps/tests.py | 135 +++++++++++++++++++++++--------------- backend/maps/user_data.py | 4 +- 2 files changed, 83 insertions(+), 56 deletions(-) diff --git a/backend/maps/tests.py b/backend/maps/tests.py index 8ab88607..ba38a9b8 100644 --- a/backend/maps/tests.py +++ b/backend/maps/tests.py @@ -1,5 +1,6 @@ import json import shutil +from typing import ClassVar import pandas as pd from django.contrib.auth.models import Permission @@ -10,7 +11,7 @@ from maps.stats import mrp_mean -TEST_DIR = 'catalog/test' + class SerializationTest(SimpleTestCase): def test_mrp_mean_returns_json_serializable_values(self): df = pd.DataFrame( @@ -28,13 +29,15 @@ def test_mrp_mean_returns_json_serializable_values(self): class FileUploadTest(TestCase): + + CATALOG_DIR: ClassVar[str] = 'catalog/test' + def setUp(self): self.client = Client() - self.url = reverse('maps-add-data') def tearDown(self): try: - shutil.rmtree(TEST_DIR) + shutil.rmtree(self.CATALOG_DIR) except OSError: pass @@ -45,104 +48,128 @@ def get_user_with_permission(self, username, password, codename): return user - def test_file_upload_add_ascii_content(self): + ########################################## + # ADD RESOURCE + ########################################## + + def add_resource_test_recipe(self, file_content: str | bytes, filename: str): user = self.get_user_with_permission("testuser", "pass", "add_data") self.client.force_login(user) - - file_content = b'col_1,col_2\nvalue1,value2' - uploaded_file = SimpleUploadedFile( - name="ascii.csv", + name=filename, content=file_content, content_type='multipart/form-data' ) - response = self.client.post( - self.url, + reverse("maps-add-data"), data={ 'file': uploaded_file, - 'package': TEST_DIR, - 'action': 'add' + 'package': self.CATALOG_DIR } ) - self.assertEqual(response.status_code, 200, f"response is {response}") self.assertEqual(uploaded_file.name, response.json()['filename']) - self.assertIn('File uploaded successfully', response.json()['message']) + self.assertIn('Resource successfully added to datapackage', response.json()['message']) + + def test_add_resource_from_file_with_ascii_content(self): + file_content = b'col_1,col_2\nvalue1,value2' + self.add_resource_test_recipe(file_content, "ascii.csv") + def test_add_resource_from_file_with_ut8_content(self): + file_content = 'col_1,col_2\néàë,-°$' + self.add_resource_test_recipe(file_content, "utf8.csv") - def test_file_upload_add_ut8_content(self): - user = self.get_user_with_permission("testuser", "pass", "add_data") - self.client.force_login(user) + def test_add_resource_from_file_with_weird_byte(self): + data = b"id,name\n1,John\n2,Ana\n3,Bob\x96\n" # 0x96 is typical Windows-1252 dash + self.add_resource_test_recipe(data, "ascii_with_weird_byte.csv") - file_content = 'col_1,col_2\néàë,-°$' + def test_file_mixed_encodings(self): + part_utf8 = "id,name,city\n1,Élodie,Paris\n".encode("utf-8") + part_latin1 = "2,José,Lisboa\n3,François,Lyon\n".encode("latin-1") + file_content = part_utf8 + part_latin1 + self.add_resource_test_recipe(file_content, "mixed_encoding_file.csv") + ########################################## + # REMOVE RESOURCE + ########################################## + + def remove_resource_test_recipe(self, file_content: str | bytes, filename: str): + user = self.get_user_with_permission("testuser", "pass", "add_data") + self.client.force_login(user) uploaded_file = SimpleUploadedFile( - name="utf8.csv", - content=file_content.encode('utf-8'), + name=filename, + content=file_content, content_type='multipart/form-data' ) - response = self.client.post( - self.url, + reverse("maps-remove-data"), data={ 'file': uploaded_file, - 'package': TEST_DIR, - 'action': 'add' + 'package': self.CATALOG_DIR } ) - self.assertEqual(response.status_code, 200, f"response is {response}") self.assertEqual(uploaded_file.name, response.json()['filename']) - self.assertIn('File uploaded successfully', response.json()['message']) - - def test_file_upload_weird_byte(self): + self.assertIn('Resource successfully removed from datapackage', response.json()['message']) + + + def test_remove_resource_from_file(self): + file_content = b'col_1,col_2\nvalue1,value2' + self.add_resource_test_recipe(file_content, "file.csv") + self.remove_resource_test_recipe(file_content, "file.csv") + + ########################################## + # APPEND DATA + ########################################## + + def append_resource_test_recipe(self, file_content: str | bytes, filename: str): user = self.get_user_with_permission("testuser", "pass", "add_data") self.client.force_login(user) - data = b"id,name\n1,John\n2,Ana\n3,Bob\x96\n" # 0x96 is typical Windows-1252 dash - uploaded_file = SimpleUploadedFile( - name="ascii_with_weird_byte.csv", - content=data, + name=filename, + content=file_content, content_type='multipart/form-data' ) - response = self.client.post( - self.url, + reverse("maps-remove-data"), data={ 'file': uploaded_file, - 'package': TEST_DIR, - 'action': 'add' + 'package': self.CATALOG_DIR } - ) - + ) self.assertEqual(response.status_code, 200, f"response is {response}") self.assertEqual(uploaded_file.name, response.json()['filename']) - self.assertIn('File uploaded successfully', response.json()['message']) - - def test_file_mixed_encodings(self): + self.assertIn('File successfully appended to datapackage resource data', response.json()['message']) + + def test_append_data_to_resource_from_file(self): + file_content = b'col_1,col_2\nvalue1,value2' + self.add_resource_test_recipe(file_content, "file.csv") + self.append_resource_test_recipe(file_content, "file.csv") + + ########################################## + # REPLACE DATA + ########################################## + + def replace_resource_test_recipe(self, file_content: str | bytes, filename: str): user = self.get_user_with_permission("testuser", "pass", "add_data") self.client.force_login(user) - part_utf8 = "id,name,city\n1,Élodie,Paris\n".encode("utf-8") - part_latin1 = "2,José,Lisboa\n3,François,Lyon\n".encode("latin-1") - file_content = part_utf8 + part_latin1 - uploaded_file = SimpleUploadedFile( - name="mixed_encoding_file.csv", + name=filename, content=file_content, content_type='multipart/form-data' ) - response = self.client.post( - self.url, + reverse("maps-remove-data"), data={ 'file': uploaded_file, - 'package': TEST_DIR, - 'action': 'add' + 'package': self.CATALOG_DIR } - ) - + ) self.assertEqual(response.status_code, 200, f"response is {response}") self.assertEqual(uploaded_file.name, response.json()['filename']) - self.assertIn('File uploaded successfully', response.json()['message']) - \ No newline at end of file + self.assertIn('File successfully replaced datapackage resource data', response.json()['message']) + + def test_replace_data_in_resource_from_file(self): + file_content = b'col_1,col_2\nvalue1,value2' + self.add_resource_test_recipe(file_content, "file.csv") + self.replace_resource_test_recipe(file_content, "file.csv") \ No newline at end of file diff --git a/backend/maps/user_data.py b/backend/maps/user_data.py index 34d11485..86f95d2f 100644 --- a/backend/maps/user_data.py +++ b/backend/maps/user_data.py @@ -4,7 +4,7 @@ import chardet from django.http import JsonResponse -from coordo.loaders import Loader, get_file_loader +from coordo.loaders import get_file_loader @@ -65,7 +65,7 @@ def replace_data(request): def update_data(request, action: str, response_message: str): """ View for updating data from a file to a resource of a DataPackage. - Expects a POST request with a body containing the 'file', 'package' and 'action' fields + Expects a POST request with a body containing the 'file' and 'package' fields. Another optional body field, named 'resource', allows specifying the target resource in case it is different from the file stem. The uploaded file is saved temporarily in the system's temporary folder, processed and removed at the end. """ From b21f91fd4b145ed9b6fe3cd7be395994d68102bb Mon Sep 17 00:00:00 2001 From: Olivier Date: Sun, 5 Jul 2026 23:37:35 +0200 Subject: [PATCH 4/4] fix tests --- backend/all4trees/settings.py | 24 +++++++++ backend/maps/tests.py | 97 +++++++++++++---------------------- backend/maps/urls.py | 4 +- backend/maps/user_data.py | 23 ++++++--- backend/maps/views.py | 4 +- 5 files changed, 81 insertions(+), 71 deletions(-) diff --git a/backend/all4trees/settings.py b/backend/all4trees/settings.py index 78eb8db6..8e44030e 100644 --- a/backend/all4trees/settings.py +++ b/backend/all4trees/settings.py @@ -160,3 +160,27 @@ "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage", # see https://whitenoise.readthedocs.io/en/stable/django.html#add-compression-and-caching-support }, } + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "console": { + "class": "logging.StreamHandler", + }, + }, + "loggers": { + "django": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + }, + "formatters": { + "simple": { + "format": "{levelname} {message}", + "style": "{", + }, + }, + +} \ No newline at end of file diff --git a/backend/maps/tests.py b/backend/maps/tests.py index ba38a9b8..113773cd 100644 --- a/backend/maps/tests.py +++ b/backend/maps/tests.py @@ -42,41 +42,54 @@ def tearDown(self): pass def get_user_with_permission(self, username, password, codename): - user = get_user_model().objects.create_user(username=username, password=password) + user, _ = get_user_model().objects.get_or_create(username=username, password=password) permission = Permission.objects.get(codename=codename) user.user_permissions.add(permission) return user - ########################################## - # ADD RESOURCE - ########################################## - - def add_resource_test_recipe(self, file_content: str | bytes, filename: str): - user = self.get_user_with_permission("testuser", "pass", "add_data") - self.client.force_login(user) - uploaded_file = SimpleUploadedFile( + def get_uplodaded_file(self, file_content: str | bytes, filename: str) -> SimpleUploadedFile: + if isinstance(file_content, str): + file_content = file_content.encode() + return SimpleUploadedFile( name=filename, content=file_content, content_type='multipart/form-data' ) - response = self.client.post( - reverse("maps-add-data"), + + + def send_post_request(self, url_name: str, file: SimpleUploadedFile): + return self.client.post( + reverse(url_name), data={ - 'file': uploaded_file, + 'file': file, 'package': self.CATALOG_DIR } ) + + def assert_all(self, response, file: SimpleUploadedFile, success_message: str): self.assertEqual(response.status_code, 200, f"response is {response}") - self.assertEqual(uploaded_file.name, response.json()['filename']) - self.assertIn('Resource successfully added to datapackage', response.json()['message']) + self.assertEqual(file.name, response.json()['filename']) + self.assertIn(success_message, response.json()['message']) + + + ########################################## + # ADD RESOURCE + ########################################## + + def add_resource_test_recipe(self, file_content: str | bytes, filename: str): + user = self.get_user_with_permission("testuser", "pass", "add_data") + self.client.force_login(user) + uploaded_file = self.get_uplodaded_file(file_content, filename) + response = self.send_post_request("maps-add-data", uploaded_file) + self.assert_all(response, uploaded_file, 'Resource successfully added to datapackage') def test_add_resource_from_file_with_ascii_content(self): file_content = b'col_1,col_2\nvalue1,value2' self.add_resource_test_recipe(file_content, "ascii.csv") def test_add_resource_from_file_with_ut8_content(self): - file_content = 'col_1,col_2\néàë,-°$' + file_content = 'col_1,col_2\néàë,-°$'.encode() self.add_resource_test_recipe(file_content, "utf8.csv") def test_add_resource_from_file_with_weird_byte(self): @@ -96,21 +109,9 @@ def test_file_mixed_encodings(self): def remove_resource_test_recipe(self, file_content: str | bytes, filename: str): user = self.get_user_with_permission("testuser", "pass", "add_data") self.client.force_login(user) - uploaded_file = SimpleUploadedFile( - name=filename, - content=file_content, - content_type='multipart/form-data' - ) - response = self.client.post( - reverse("maps-remove-data"), - data={ - 'file': uploaded_file, - 'package': self.CATALOG_DIR - } - ) - self.assertEqual(response.status_code, 200, f"response is {response}") - self.assertEqual(uploaded_file.name, response.json()['filename']) - self.assertIn('Resource successfully removed from datapackage', response.json()['message']) + uploaded_file = self.get_uplodaded_file(file_content, filename) + response = self.send_post_request("maps-remove-data", uploaded_file) + self.assert_all(response, uploaded_file, 'Resource successfully removed from datapackage') def test_remove_resource_from_file(self): @@ -125,21 +126,9 @@ def test_remove_resource_from_file(self): def append_resource_test_recipe(self, file_content: str | bytes, filename: str): user = self.get_user_with_permission("testuser", "pass", "add_data") self.client.force_login(user) - uploaded_file = SimpleUploadedFile( - name=filename, - content=file_content, - content_type='multipart/form-data' - ) - response = self.client.post( - reverse("maps-remove-data"), - data={ - 'file': uploaded_file, - 'package': self.CATALOG_DIR - } - ) - self.assertEqual(response.status_code, 200, f"response is {response}") - self.assertEqual(uploaded_file.name, response.json()['filename']) - self.assertIn('File successfully appended to datapackage resource data', response.json()['message']) + uploaded_file = self.get_uplodaded_file(file_content, filename) + response = self.send_post_request("maps-append-data", uploaded_file) + self.assert_all(response, uploaded_file, 'File successfully appended to datapackage resource data') def test_append_data_to_resource_from_file(self): file_content = b'col_1,col_2\nvalue1,value2' @@ -153,21 +142,9 @@ def test_append_data_to_resource_from_file(self): def replace_resource_test_recipe(self, file_content: str | bytes, filename: str): user = self.get_user_with_permission("testuser", "pass", "add_data") self.client.force_login(user) - uploaded_file = SimpleUploadedFile( - name=filename, - content=file_content, - content_type='multipart/form-data' - ) - response = self.client.post( - reverse("maps-remove-data"), - data={ - 'file': uploaded_file, - 'package': self.CATALOG_DIR - } - ) - self.assertEqual(response.status_code, 200, f"response is {response}") - self.assertEqual(uploaded_file.name, response.json()['filename']) - self.assertIn('File successfully replaced datapackage resource data', response.json()['message']) + uploaded_file = self.get_uplodaded_file(file_content, filename) + response = self.send_post_request("maps-replace-data", uploaded_file) + self.assert_all(response, uploaded_file, 'File successfully replaced datapackage resource data') def test_replace_data_in_resource_from_file(self): file_content = b'col_1,col_2\nvalue1,value2' diff --git a/backend/maps/urls.py b/backend/maps/urls.py index 89b14e44..93f0ff52 100644 --- a/backend/maps/urls.py +++ b/backend/maps/urls.py @@ -3,8 +3,8 @@ from . import views urlpatterns = [ - path("add-data/", views.add_resource_using_file_view, name="maps-add-data"), - path("remove-data/", views.remove_resource_using_file_view, name="maps-remove-data"), + path("add-data/", views.add_resource_from_file_view, name="maps-add-data"), + path("remove-data/", views.remove_resource_from_file_view, name="maps-remove-data"), path("append-data/", views.append_file_data_to_datapackage_view, name="maps-append-data"), path("replace-data/", views.replace_datapackage_data_from_file_view, name="maps-replace-data"), re_path(r"^(?!dashboard)(?P\w+)", views.my_map_view, name="maps-data"), diff --git a/backend/maps/user_data.py b/backend/maps/user_data.py index 86f95d2f..d49e1fe4 100644 --- a/backend/maps/user_data.py +++ b/backend/maps/user_data.py @@ -1,5 +1,6 @@ from tempfile import gettempdir from pathlib import Path +import logging import chardet @@ -7,9 +8,11 @@ from coordo.loaders import get_file_loader +logger = logging.getLogger(__name__) + def add_resource(request): - return modify_resource(request, "remove", "Resource successfully added to datapackage") + return modify_resource(request, "add", "Resource successfully added to datapackage") def remove_resource(request): @@ -28,7 +31,8 @@ def modify_resource(request, action: str, response_message: str): try: package = Path(request.POST["package"]) options: dict = request.POST.get("options", {}) - except KeyError: + except KeyError as e: + logger.exception(e) return JsonResponse( {"error": "Invalid request format. The request must contain at least the 'package' field."}, status=400 @@ -41,6 +45,7 @@ def modify_resource(request, action: str, response_message: str): elif action == "remove": file_loader.remove() except Exception as e: + logger.exception(e) return JsonResponse({'error': str(e)}, status=500) finally: @@ -50,7 +55,7 @@ def modify_resource(request, action: str, response_message: str): return JsonResponse({ 'message': response_message, 'filename': file.name, - 'package': package + 'package': str(package) }) @@ -76,7 +81,8 @@ def update_data(request, action: str, response_message: str): package = Path(request.POST["package"]) options: dict = request.POST.get("options", {}) target_resource_name: str = request.POST.get("resource", None) - except KeyError: + except KeyError as e: + logger.exception(e) return JsonResponse( {"error": "Invalid request format. The request must contain at least the 'package' field."}, status=400 @@ -93,16 +99,17 @@ def update_data(request, action: str, response_message: str): else: raise ValueError(f"Invalid action name: {action}. It must be one of: 'append' or 'replace'") except Exception as e: + logger.exception(e) return JsonResponse({'error': str(e)}, status=500) finally: # in any case, delete the temporary file file.unlink(missing_ok=True) - + return JsonResponse({ 'message': response_message, 'filename': file.name, - 'package': package + 'package': str(package) }) @@ -113,7 +120,9 @@ def get_data_file(request) -> Path: Writes the file to a temporary file in the system's temporary folder. """ if 'file' not in request.FILES: - return JsonResponse({'error': 'No file provided'}, status=400) + msg = 'No file provided' + logger.error(msg) + return JsonResponse({'error': msg}, status=400) uploaded_file = request.FILES['file'] temp_file = Path(gettempdir()) / uploaded_file.name diff --git a/backend/maps/views.py b/backend/maps/views.py index 6d46140b..dddcf1de 100644 --- a/backend/maps/views.py +++ b/backend/maps/views.py @@ -38,7 +38,7 @@ def dashboard_view(request, layer_id): @require_POST @permission_required("users.add_data") -def add_resource_using_file_view(request): +def add_resource_from_file_view(request): """ View for adding a file (creating the corresponding resource(s)) to a DataPackage. Expects a POST request with a body containing the 'file', 'package' fields. @@ -49,7 +49,7 @@ def add_resource_using_file_view(request): @require_POST @permission_required("users.add_data") -def remove_resource_using_file_view(request): +def remove_resource_from_file_view(request): """ View for removing resource(s) cooresponding a file to a DataPackage. Expects a POST request with a body containing the 'file', 'package' fields.