Update Makefile to adapt to coordo CLI update

backend/Makefile

@@ -1,55 +1,55 @@
 catalog:
 	# All 4 Trees - Inventaire forestier
 
-	coordo load kobotoolbox --action add data/all4trees/inventaire_for/20260519_InventaireForestier_QuestionnaireK.xlsx data/all4trees/inventaire_for/20260422_InventaireForestier_DonneesK.xlsx --package catalog/inventaire_for
-	coordo load file --action add data/all4trees/inventaire_for/20260422_InventaireForestier_DonneesExternes.xlsx --package catalog/inventaire_for
-
-	coordo add-foreignkey adu.decay for_dw.decay --package catalog/inventaire_for
-	coordo add-foreignkey inv_for.proj for_samp.proj --package catalog/inventaire_for
-	coordo add-foreignkey inv_for.typ for_pop.typ --package catalog/inventaire_for
-	coordo add-foreignkey inv_for.loc2 for_weath.loc2 --package catalog/inventaire_for
-	coordo add-foreignkey inv_for._id for_soil._index --package catalog/inventaire_for
-	coordo add-foreignkey adu.adu_sp for_sp.adu_sp --package catalog/inventaire_for
-	coordo add-foreignkey tsbf_001.tsbf_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for
-	coordo add-foreignkey barba_001.barbA_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for
-	coordo add-foreignkey barbb_001.barbB_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for
-	coordo add-foreignkey barbc_001.barbC_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for
-	coordo add-foreignkey barbd_001.barbD_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for
-	coordo add-foreignkey tsbf_001.tsbf_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for
-	coordo add-foreignkey barba_001.barbA_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for
-	coordo add-foreignkey barbb_001.barbB_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for
-	coordo add-foreignkey barbc_001.barbC_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for
-	coordo add-foreignkey barbd_001.barbD_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for
-	coordo add-foreignkey tsbf_001.tsbf_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for
-	coordo add-foreignkey barba_001.barbA_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for
-	coordo add-foreignkey barbb_001.barbB_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for
-	coordo add-foreignkey barbc_001.barbC_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for
-	coordo add-foreignkey barbd_001.barbD_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for
+	coordo add kobotoolbox data/all4trees/inventaire_for/20260519_InventaireForestier_QuestionnaireK.xlsx data/all4trees/inventaire_for/20260422_InventaireForestier_DonneesK.xlsx --package catalog/inventaire_for
+	coordo add file data/all4trees/inventaire_for/20260422_InventaireForestier_DonneesExternes.xlsx --package catalog/inventaire_for
+
+	coordo add foreignkey adu.decay for_dw.decay --package catalog/inventaire_for
+	coordo add foreignkey inv_for.proj for_samp.proj --package catalog/inventaire_for
+	coordo add foreignkey inv_for.typ for_pop.typ --package catalog/inventaire_for
+	coordo add foreignkey inv_for.loc2 for_weath.loc2 --package catalog/inventaire_for
+	coordo add foreignkey inv_for._id for_soil._index --package catalog/inventaire_for
+	coordo add foreignkey adu.adu_sp for_sp.adu_sp --package catalog/inventaire_for
+	coordo add foreignkey tsbf_001.tsbf_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for
+	coordo add foreignkey barba_001.barbA_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for
+	coordo add foreignkey barbb_001.barbB_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for
+	coordo add foreignkey barbc_001.barbC_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for
+	coordo add foreignkey barbd_001.barbD_tax1 for_mf_tax1.tax1 --package catalog/inventaire_for
+	coordo add foreignkey tsbf_001.tsbf_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for
+	coordo add foreignkey barba_001.barbA_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for
+	coordo add foreignkey barbb_001.barbB_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for
+	coordo add foreignkey barbc_001.barbC_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for
+	coordo add foreignkey barbd_001.barbD_tax2 for_mf_tax2.tax2 --package catalog/inventaire_for
+	coordo add foreignkey tsbf_001.tsbf_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for
+	coordo add foreignkey barba_001.barbA_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for
+	coordo add foreignkey barbb_001.barbB_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for
+	coordo add foreignkey barbc_001.barbC_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for
+	coordo add foreignkey barbd_001.barbD_tax3 for_mf_tax3.tax3 --package catalog/inventaire_for
 
 	# All 4 Trees - Inventaire Biologique
-	coordo load kobotoolbox --action add data/all4trees/inventaire_bio/20260519_InventaireBiologique_QuestionnaireK.xlsx data/all4trees/inventaire_bio/20260422_InventaireBiologique_DonneesK.xlsx --package catalog/inventaire_bio
-	coordo load file --action add data/all4trees/inventaire_bio/20260422_InventaireBiologique_DonneesExternes.xlsx --package catalog/inventaire_bio
+	coordo add kobotoolbox data/all4trees/inventaire_bio/20260519_InventaireBiologique_QuestionnaireK.xlsx data/all4trees/inventaire_bio/20260422_InventaireBiologique_DonneesK.xlsx --package catalog/inventaire_bio
+	coordo add file data/all4trees/inventaire_bio/20260422_InventaireBiologique_DonneesExternes.xlsx --package catalog/inventaire_bio
 
-	coordo add-foreignkey inv_bio.proj bio_samp.proj --package catalog/inventaire_bio
-	coordo add-foreignkey inv_bio.year bio_pop.year --package catalog/inventaire_bio
-	coordo add-foreignkey inv_001.tax3 bio_sp.tax3 --package catalog/inventaire_bio
+	coordo add foreignkey inv_bio.proj bio_samp.proj --package catalog/inventaire_bio
+	coordo add foreignkey inv_bio.year bio_pop.year --package catalog/inventaire_bio
+	coordo add foreignkey inv_001.tax3 bio_sp.tax3 --package catalog/inventaire_bio
 
 	# All 4 Trees - Enquête ménage
-	coordo load kobotoolbox --action add data/all4trees/enquete/20260519_EnqueteMenage_QuestionnaireK.xlsx data/all4trees/enquete/20260422_EnqueteMenage_DonneesK.csv --package catalog/enquete
-	coordo load file --action add data/all4trees/enquete/20260422_EnqueteMenage_DonneesExternes.xlsx --package catalog/enquete
+	coordo add kobotoolbox data/all4trees/enquete/20260519_EnqueteMenage_QuestionnaireK.xlsx data/all4trees/enquete/20260422_EnqueteMenage_DonneesK.csv --package catalog/enquete
+	coordo add file data/all4trees/enquete/20260422_EnqueteMenage_DonneesExternes.xlsx --package catalog/enquete
 
-	coordo add-foreignkey enquete_menage.proj enq_samp.proj --package catalog/enquete
-	coordo add-foreignkey enquete_menage.typ enq_pop.typ --package catalog/enquete
-	coordo add-foreignkey enquete_menage.loc2 enq_gps.loc2 --package catalog/enquete
-	coordo add-foreignkey enquete_menage.fw_cod enq_fw.fw_cod --package catalog/enquete
-	coordo add-foreignkey enquete_menage.coal_unit enq_coal.coal_unit --package catalog/enquete
+	coordo add foreignkey enquete_menage.proj enq_samp.proj --package catalog/enquete
+	coordo add foreignkey enquete_menage.typ enq_pop.typ --package catalog/enquete
+	coordo add foreignkey enquete_menage.loc2 enq_gps.loc2 --package catalog/enquete
+	coordo add foreignkey enquete_menage.fw_cod enq_fw.fw_cod --package catalog/enquete
+	coordo add foreignkey enquete_menage.coal_unit enq_coal.coal_unit --package catalog/enquete
 
 	# Seed - Survey
-	coordo load file --action add data/seed/survey.zip --package catalog/seed
-	coordo load file --action add data/seed/suivi_sites_reboises.zip --package catalog/seed
-	coordo load file --action add data/seed/ident_sites.zip --package catalog/seed
-	coordo load file --action add data/seed/reboisement_id.zip --package catalog/seed
-	coordo load file --action add data/seed/sensib_communautaire.zip --package catalog/seed
+	coordo add file data/seed/survey.zip --package catalog/seed
+	coordo add file data/seed/suivi_sites_reboises.zip --package catalog/seed
+	coordo add file data/seed/ident_sites.zip --package catalog/seed
+	coordo add file data/seed/reboisement_id.zip --package catalog/seed
+	coordo add file data/seed/sensib_communautaire.zip --package catalog/seed
 
 test:
 	python manage.py test

backend/all4trees/settings.py

@@ -160,3 +160,27 @@
         "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage",  # see https://whitenoise.readthedocs.io/en/stable/django.html#add-compression-and-caching-support
     },
 }
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+        },
+    },
+    "loggers": {
+        "django": {
+            "handlers": ["console"],
+            "level": "INFO",
+            "propagate": False,
+        },
+    },
+    "formatters": {
+        "simple": {
+            "format": "{levelname} {message}",
+            "style": "{",
+        },
+    },
+
+}

backend/maps/tests.py

@@ -1,5 +1,6 @@
 import json
 import shutil
+from typing import ClassVar
 
 import pandas as pd
 from django.contrib.auth.models import Permission
@@ -10,7 +11,7 @@
 
 from maps.stats import mrp_mean
 
-TEST_DIR = 'catalog/test'
+
 class SerializationTest(SimpleTestCase):
     def test_mrp_mean_returns_json_serializable_values(self):
         df = pd.DataFrame(
@@ -28,121 +29,124 @@ def test_mrp_mean_returns_json_serializable_values(self):
 
 
 class FileUploadTest(TestCase):
+
+    CATALOG_DIR: ClassVar[str] = 'catalog/test'
+
     def setUp(self):
         self.client = Client()
-        self.url = reverse('maps-add-data')
 
     def tearDown(self):
         try:
-            shutil.rmtree(TEST_DIR)
+            shutil.rmtree(self.CATALOG_DIR)
         except OSError:
             pass 
 
     def get_user_with_permission(self, username, password, codename):
-        user = get_user_model().objects.create_user(username=username, password=password)
+        user, _ = get_user_model().objects.get_or_create(username=username, password=password)
         permission = Permission.objects.get(codename=codename)
         user.user_permissions.add(permission)
         return user
 
 
-    def test_file_upload_add_ascii_content(self):
-        user = self.get_user_with_permission("testuser", "pass", "add_data")
-        self.client.force_login(user)
-
-        file_content = b'col_1,col_2\nvalue1,value2'
-
-        uploaded_file = SimpleUploadedFile(
-            name="ascii.csv",
+    def get_uplodaded_file(self, file_content: str | bytes, filename: str) -> SimpleUploadedFile:
+        if isinstance(file_content, str):
+            file_content = file_content.encode()
+        return SimpleUploadedFile(
+            name=filename,
             content=file_content,
             content_type='multipart/form-data'
         )
 
-        response = self.client.post(
-            self.url,
+
+    def send_post_request(self, url_name: str, file: SimpleUploadedFile):
+        return self.client.post(
+            reverse(url_name),
             data={ 
-                'file': uploaded_file, 
-                'package': TEST_DIR,
-                'action': 'add'
+                'file': file, 
+                'package': self.CATALOG_DIR
             }
         )
-
+
+    def assert_all(self, response, file: SimpleUploadedFile, success_message: str):
         self.assertEqual(response.status_code, 200, f"response is {response}")
-        self.assertEqual(uploaded_file.name, response.json()['filename'])
-        self.assertIn('File uploaded successfully', response.json()['message'])
+        self.assertEqual(file.name, response.json()['filename'])
+        self.assertIn(success_message, response.json()['message'])
 
 
-    def test_file_upload_add_ut8_content(self):
+    ##########################################
+    # ADD RESOURCE
+    ##########################################
+
+    def add_resource_test_recipe(self, file_content: str | bytes, filename: str):
         user = self.get_user_with_permission("testuser", "pass", "add_data")
         self.client.force_login(user)
+        uploaded_file = self.get_uplodaded_file(file_content, filename)
+        response = self.send_post_request("maps-add-data", uploaded_file)
+        self.assert_all(response, uploaded_file, 'Resource successfully added to datapackage')
 
-        file_content = 'col_1,col_2\néàë,-°$'
+    def test_add_resource_from_file_with_ascii_content(self):
+        file_content = b'col_1,col_2\nvalue1,value2'
+        self.add_resource_test_recipe(file_content, "ascii.csv")
 
-        uploaded_file = SimpleUploadedFile(
-            name="utf8.csv",
-            content=file_content.encode('utf-8'),
-            content_type='multipart/form-data'
-        )
+    def test_add_resource_from_file_with_ut8_content(self):
+        file_content = 'col_1,col_2\néàë,-°$'.encode()
+        self.add_resource_test_recipe(file_content, "utf8.csv")
 
-        response = self.client.post(
-            self.url,
-            data={ 
-                'file': uploaded_file, 
-                'package': TEST_DIR,
-                'action': 'add'
-            }
-        )
-
-        self.assertEqual(response.status_code, 200, f"response is {response}")
-        self.assertEqual(uploaded_file.name, response.json()['filename'])
-        self.assertIn('File uploaded successfully', response.json()['message'])
+    def test_add_resource_from_file_with_weird_byte(self):
+        data = b"id,name\n1,John\n2,Ana\n3,Bob\x96\n"  # 0x96 is typical Windows-1252 dash
+        self.add_resource_test_recipe(data, "ascii_with_weird_byte.csv")
 
-    def test_file_upload_weird_byte(self):
+    def test_file_mixed_encodings(self):
+        part_utf8 = "id,name,city\n1,Élodie,Paris\n".encode("utf-8")
+        part_latin1 = "2,José,Lisboa\n3,François,Lyon\n".encode("latin-1")
+        file_content = part_utf8 + part_latin1
+        self.add_resource_test_recipe(file_content, "mixed_encoding_file.csv")
+
+    ##########################################
+    # REMOVE RESOURCE
+    ##########################################
+
+    def remove_resource_test_recipe(self, file_content: str | bytes, filename: str):
         user = self.get_user_with_permission("testuser", "pass", "add_data")
         self.client.force_login(user)
-        data = b"id,name\n1,John\n2,Ana\n3,Bob\x96\n"  # 0x96 is typical Windows-1252 dash
+        uploaded_file = self.get_uplodaded_file(file_content, filename)
+        response = self.send_post_request("maps-remove-data", uploaded_file)
+        self.assert_all(response, uploaded_file, 'Resource successfully removed from datapackage')
 
-        uploaded_file = SimpleUploadedFile(
-            name="ascii_with_weird_byte.csv",
-            content=data,
-            content_type='multipart/form-data'
-        )
-
-        response = self.client.post(
-            self.url,
-            data={ 
-                'file': uploaded_file, 
-                'package': TEST_DIR,
-                'action': 'add'
-            }
-            )
 
-        self.assertEqual(response.status_code, 200, f"response is {response}")
-        self.assertEqual(uploaded_file.name, response.json()['filename'])
-        self.assertIn('File uploaded successfully', response.json()['message'])
-
-    def test_file_mixed_encodings(self):
+    def test_remove_resource_from_file(self):
+        file_content = b'col_1,col_2\nvalue1,value2'
+        self.add_resource_test_recipe(file_content, "file.csv")
+        self.remove_resource_test_recipe(file_content, "file.csv")
+
+    ##########################################
+    # APPEND DATA
+    ##########################################
+
+    def append_resource_test_recipe(self, file_content: str | bytes, filename: str):
         user = self.get_user_with_permission("testuser", "pass", "add_data")
         self.client.force_login(user)
-        part_utf8 = "id,name,city\n1,Élodie,Paris\n".encode("utf-8")
-        part_latin1 = "2,José,Lisboa\n3,François,Lyon\n".encode("latin-1")
-        file_content = part_utf8 + part_latin1
+        uploaded_file = self.get_uplodaded_file(file_content, filename)
+        response = self.send_post_request("maps-append-data", uploaded_file)
+        self.assert_all(response, uploaded_file, 'File successfully appended to datapackage resource data')
 
-        uploaded_file = SimpleUploadedFile(
-            name="mixed_encoding_file.csv",
-            content=file_content,
-            content_type='multipart/form-data'
-        )
-
-        response = self.client.post(
-            self.url,
-            data={ 
-                'file': uploaded_file, 
-                'package': TEST_DIR,
-                'action': 'add'
-            }
-            )
+    def test_append_data_to_resource_from_file(self):
+        file_content = b'col_1,col_2\nvalue1,value2'
+        self.add_resource_test_recipe(file_content, "file.csv")
+        self.append_resource_test_recipe(file_content, "file.csv")
 
-        self.assertEqual(response.status_code, 200, f"response is {response}")
-        self.assertEqual(uploaded_file.name, response.json()['filename'])
-        self.assertIn('File uploaded successfully', response.json()['message'])
-
+    ##########################################
+    # REPLACE DATA
+    ##########################################
+
+    def replace_resource_test_recipe(self, file_content: str | bytes, filename: str):
+        user = self.get_user_with_permission("testuser", "pass", "add_data")
+        self.client.force_login(user)
+        uploaded_file = self.get_uplodaded_file(file_content, filename)
+        response = self.send_post_request("maps-replace-data", uploaded_file)
+        self.assert_all(response, uploaded_file, 'File successfully replaced datapackage resource data')
+
+    def test_replace_data_in_resource_from_file(self):
+        file_content = b'col_1,col_2\nvalue1,value2'
+        self.add_resource_test_recipe(file_content, "file.csv")
+        self.replace_resource_test_recipe(file_content, "file.csv")

backend/maps/urls.py

@@ -3,7 +3,10 @@
 from . import views
 
 urlpatterns = [
-    path("add-data/", views.add_data_view, name="maps-add-data"),
+    path("add-data/", views.add_resource_from_file_view, name="maps-add-data"),
+    path("remove-data/", views.remove_resource_from_file_view, name="maps-remove-data"),
+    path("append-data/", views.append_file_data_to_datapackage_view, name="maps-append-data"),
+    path("replace-data/", views.replace_datapackage_data_from_file_view, name="maps-replace-data"),
     re_path(r"^(?!dashboard)(?P<subpath>\w+)", views.my_map_view, name="maps-data"),
     path("dashboard/<layer_id>", views.dashboard_view, name="dashboard-data"),
 ]