Make the button "Download job.zip" available for all calculations

bin/run-demos.sh

@@ -83,5 +83,19 @@ oq info usgs_rupture:us70006sj8
 # display the calculations
 oq db find %
 
+echo "Testing job exporter for all generated jobs"
+failed_export_job_zips=()
+for job_id in $(oq db "SELECT id FROM job" | grep -oE '[0-9]+'); do
+    echo "--> Testing export job zip for job ID: ${job_id}"
+    if ! oq export "job" -e zip ${job_id}; then
+        failed_export_job_zips+=("${job_id}")
+    fi
+done
+
+if [ ${#failed_export_job_zips[@]} -gt 0 ]; then
+    echo "Export test failed for job ID(s): ${failed_export_job_zips[*]}"
+    exit 1
+fi
+
 # build an HTML report
 oq engine --make-report today

openquake/calculators/export/risk.py

@@ -22,8 +22,9 @@
 import collections
 import numpy
 import pandas
+import tempfile
 
-from openquake.baselib import hdf5, writers, general, node
+from openquake.baselib import config, hdf5, writers, general, node
 from openquake.baselib.general import decode
 from openquake.hazardlib import nrml
 from openquake.hazardlib.stats import compute_stats2, mean_curve
@@ -797,8 +798,12 @@ def convert_df_to_vulnerability(loss_type, df):
     descr = N('description', {}, f"{loss_type} vulnerability model")
     root.append(descr)
     for riskfunc in df.riskfunc:
-        rfunc = json.loads(riskfunc)[
-            'openquake.risklib.scientific.VulnerabilityFunction']
+        dic = json.loads(riskfunc)
+        key = 'openquake.risklib.scientific.VulnerabilityFunction'
+        if key not in dic:
+            # Skip non-vulnerability functions (e.g. FragilityFunctionList)
+            continue
+        rfunc = dic[key]
         vfunc = N('vulnerabilityFunction',
                   {'id': rfunc['id'], 'dist': rfunc['distribution_name']})
         imls = N('imls', {'imt': rfunc['imt']}, rfunc['imls'])
@@ -922,22 +927,7 @@ def export_exposure(ekey, dstore):
     return [exposure_xml, assetcol_csv]
 
 
-# tested in impact_test[1]
-@export.add(('job', 'zip'))
-def export_job_zip(ekey, dstore):
-    """
-    Exports:
-    - job.ini
-    - rupture.csv
-    - gsim_lt.xml
-    - site_model.csv
-    - exposure.xml and assetcol.csv
-    - vulnerability functions.xml
-    - taxonomy_mapping.csv
-    - consequences.csv
-    """
-    inputs = {}
-    oq = dstore['oqparam']
+def _get_gsim_lt(dstore, oq, inputs):
     if oq.shakemap_uri or 'usgs_id' in oq.rupture_dict:  # from shakemap
         df = dstore.read_df('gmf_data').sort_values(['eid', 'sid'])
         ren = {'sid': 'site_id', 'eid': 'event_id'}
@@ -954,28 +944,93 @@ def export_job_zip(ekey, dstore):
             oq.rupture_dict.pop('mmi_file', None)
             oq.inputs.pop('rupture', None)
             oq.inputs.pop('mmi', None)
-        gsim_lt = None  # from shakemap
-    else:
+        return None  # from shakemap
+    elif 'ruptures' in dstore and len(dstore['ruptures']) > 0:
         model = dstore['ruptures'][0]['model'].decode('ascii')
         # FIXME: extracts the gsim_lt of the first model only
         [(model, lt)] = base.get_model_lts(dstore, model)
-        gsim_lt = lt.gsim_lt
+        return lt.gsim_lt
+    return None
+
+
+def _export_taxmap_and_consequences(dstore, oq, ddic, inputs):
+    if 'taxmap' in dstore and 'assetcol/tagcol/taxonomy' in dstore:
+        writer = writers.CsvWriter(fmt=writers.FIVEDIGITS)
+        dest = dstore.export_path('taxonomy_mapping.csv')
+        taxmap = dstore.read_df('taxmap')
+        taxonomies = dstore['assetcol/tagcol/taxonomy'][:]
+        taxmap['taxonomy'] = decode(taxonomies[taxmap['taxi']])
+        del taxmap['taxi']
+        writer.save(taxmap, dest)
+        inputs['taxonomy_mapping'] = dest
+
+    if 'consequence' in oq.inputs:
+        writer = writers.CsvWriter(fmt=writers.FIVEDIGITS)
+        consdict = readinput.read_consdict(oq, oq.limit_states, list(ddic))
+        dic = {}
+        for name_by_key, df in consdict.items():
+            name, key = name_by_key.split('_by_')
+            df['consequence'] = name
+            dest = dstore.export_path(f'consequence_{name_by_key}.csv')
+            writer.save(df, dest)
+            dic[name_by_key] = dest
+        inputs['consequence'] = dic
+
+
+# tested in impact_test[1]
+@export.add(('job', 'zip'))
+def export_job_zip(ekey, dstore):
+    """
+    Exports:
+    - job.ini
+    - rupture.csv (if present)
+    - gsim_lt.xml (if present)
+    - site_model.csv (if present)
+    - exposure.xml and assetcol.csv (if present)
+    - vulnerability/fragility functions.xml (if present)
+    - taxonomy_mapping.csv (if present)
+    - consequences.csv (if present)
+    """
+    # Ensure dstore export path resolves to a valid temp dir for restored jobs
+    if not os.path.exists(dstore.export_dir):
+        dstore.export_dir = os.path.join(
+            config.directory.custom_tmp or tempfile.gettempdir())
+    inputs = {}
+    oq = dstore['oqparam']
+    ddic = {}
+
+    gsim_lt = _get_gsim_lt(dstore, oq, inputs)
+
     oq.base_path = os.path.abspath('.')
     job_ini = dstore.export_path('%s.ini' % ekey[0])
     inputs['job_ini'] = job_ini
-    [exposure_xml, assetcol_csv] = export_exposure(('exposure', 'zip'), dstore)
-    inputs['exposure'] = exposure_xml
-    csv = extract(dstore, 'ruptures?slice=0&slice=1').array
-    if len(csv.splitlines()) > 2:  # comment + header + data
-        dest = dstore.export_path('rupture.csv')
-        with open(dest, 'w', encoding='utf8') as out:
-            out.write(csv)
-        inputs['rupture_model'] = dest
+
+    assetcol_csv = None
+    if 'assetcol' in dstore or 'exposure' in dstore:
+        try:
+            [exposure_xml, assetcol_csv] = export_exposure(
+                ('exposure', 'zip'), dstore)
+            inputs['exposure'] = exposure_xml
+        except KeyError:
+            pass
+
+    if 'ruptures' in dstore:
+        try:
+            csv = extract(dstore, 'ruptures?slice=0&slice=1').array
+            if len(csv.splitlines()) > 2:  # comment + header + data
+                dest = dstore.export_path('rupture.csv')
+                with open(dest, 'w', encoding='utf8') as out:
+                    out.write(csv)
+                inputs['rupture_model'] = dest
+        except Exception:
+            pass
+
     if gsim_lt and (oq.gsim is None or oq.gsim == '[FromFile]'):
         dest = dstore.export_path('gsim_logic_tree.xml')
         with open(dest, 'wb') as out:
             nrml.write([gsim_lt.to_node()], out)
         inputs['gsim_logic_tree'] = dest
+
     if oq.calculation_mode.endswith('risk'):
         inputs.update(export_vulnerability_xml(dstore))
     elif oq.calculation_mode.endswith('damage'):
@@ -986,33 +1041,23 @@ def export_job_zip(ekey, dstore):
                 # needed for PAPERS
                 oq.inputs.pop(f'{ltype}_fragility', None)
 
-    writer = writers.CsvWriter(fmt=writers.FIVEDIGITS)
-    dest = dstore.export_path('taxonomy_mapping.csv')
-    taxmap = dstore.read_df('taxmap')
-    taxonomies = dstore['assetcol/tagcol/taxonomy'][:]
-    taxmap['taxonomy'] = decode(taxonomies[taxmap['taxi']])
-    del taxmap['taxi']
-    writer.save(taxmap, dest)
-    inputs['taxonomy_mapping'] = dest
-    if 'consequence' in oq.inputs:
-        consdict = readinput.read_consdict(oq, oq.limit_states, list(ddic))
-        dic = {}
-        for name_by_key, df in consdict.items():
-            name, key = name_by_key.split('_by_')
-            df['consequence'] = name
-            dest = dstore.export_path(f'consequence_{name_by_key}.csv')
-            writer.save(df, dest)
-            dic[name_by_key] = dest
-        inputs['consequence'] = dic
-    inputs['site_model'] = dstore.export_path('sites.csv')
-    sitecol = dstore['sitecol']
-    sitecol.make_complete()  # needed for test_impact[1]
-    writer.save(sitecol.array, inputs['site_model'])
+    _export_taxmap_and_consequences(dstore, oq, ddic, inputs)
+
+    if 'sitecol' in dstore:
+        writer = writers.CsvWriter(fmt=writers.FIVEDIGITS)
+        inputs['site_model'] = dstore.export_path('sites.csv')
+        sitecol = dstore['sitecol']
+        sitecol.make_complete()  # needed for test_impact[1]
+        writer.save(sitecol.array, inputs['site_model'])
+
     with open(job_ini, 'w', encoding='utf8') as out:
         if 'gmfs' in inputs:
             oq.hazard_calculation_id = None
         out.write(oq.to_ini(**inputs))
-    fnames = list(inputs.values()) + [assetcol_csv]
+
+    # Filter out None values before flattening file paths
+    fnames = [f for f in list(inputs.values()) + [assetcol_csv]
+              if f is not None]
     return flatten(fnames)
 
 

openquake/commonlib/readinput.py

@@ -1158,7 +1158,16 @@ def read_consdict(oqparam, limit_states, perils):
             fnames = [fnames]
         # i.e. files collapsed.csv, fatalities.csv, ... with headers like
         # taxonomy,consequence,slight,moderate,extensive
-        df = pandas.concat([pandas.read_csv(fname) for fname in fnames])
+        dfs = []
+        for fname in fnames:
+            if os.path.exists(fname):
+                dfs.append(pandas.read_csv(fname))
+            else:
+                logging.warning(
+                    "Consequence file not found, skipping: %s", fname)
+        if not dfs:
+            continue
+        df = pandas.concat(dfs)
         # NB: consequence files depend on loss_type, unlike fragility files
         if 'taxonomy' in df.columns:  # obsolete name
             df['risk_id'] = df['taxonomy']

openquake/server/templates/engine/get_outputs.html

@@ -112,11 +112,15 @@ <h2>Outputs from calculation {{ calc_id }}</h2>
             </div>
             <div class="outputs-general-btns-right">
               {% if user_level >= 2 %}
-              <div id="my-datastore">
+              <div id="my-datastore" class="bottom-right-btns">
                 <a href="{{ oq_engine_server_url }}/v1/calc/{{ calc_id }}/datastore"
                    title="Size: {{ size_mb }} MB" class="btn btn-sm">
                   Download hdf5 datastore</a>
               </div>
+              <div id="my-job" class="bottom-right-btns">
+                <a href="{{ oq_engine_server_url }}/v1/calc/{{ calc_id }}/job_zip" class="btn btn-sm">
+                  Download job.zip</a>
+              </div>
               {% endif %} {# end if user_level >= 2 #}
             </div>
           </div>