From a788cdcf6ec4aa81a49cccff3f65e0f2dfb4594a Mon Sep 17 00:00:00 2001 From: Mike Rosseel Date: Wed, 15 Jul 2026 14:48:09 +0200 Subject: [PATCH 01/10] feat(catalog): add observable asteroids --- default_config.json | 4 +- docs/adr/0021-mpc-bright-asteroid-catalog.md | 52 +++ docs/ax/catalog.md | 46 +- docs/ax/catalog/CONTEXT.md | 20 +- docs/ax/catalog/obslist-formats/README.md | 1 + markers/mrk_asteroid.png | Bin 0 -> 126 bytes python/PiFinder/asteroid_catalog.py | 291 ++++++++++++ python/PiFinder/asteroids.py | 414 ++++++++++++++++++ python/PiFinder/catalog_base.py | 11 + python/PiFinder/catalogs.py | 40 +- python/PiFinder/comet_catalog.py | 118 +++-- python/PiFinder/comets.py | 110 +---- python/PiFinder/composite_object.py | 12 +- python/PiFinder/config.py | 26 +- python/PiFinder/download_utils.py | 141 ++++++ python/PiFinder/obj_types.py | 2 + python/PiFinder/obslist_formats.py | 7 +- python/PiFinder/ui/marking_menus.py | 1 + python/PiFinder/ui/menu_structure.py | 10 + python/PiFinder/ui/object_list.py | 284 +++++++----- python/PiFinder/utils.py | 1 + python/tests/data/asteroids_fixture.txt | 10 + python/tests/test_asteroids.py | 135 ++++++ python/tests/test_catalog_filter_cache.py | 35 +- .../tests/test_config_asteroid_migration.py | 47 ++ python/tests/test_download_utils.py | 103 +++++ python/tests/test_dynamic_catalogs.py | 143 ++++++ python/tests/test_menu_struct.py | 1 + python/tests/test_obj_types_docs.py | 13 +- python/tests/test_object_list_progress.py | 47 ++ python/tests/test_object_list_sorting.py | 68 +++ python/tests/test_obslist_formats.py | 2 + python/tests/test_ui_modules.py | 27 +- python/tests/website/test_web_remote.py | 8 +- .../tests/website/test_web_remote_objects.py | 49 ++- 35 files changed, 1978 insertions(+), 301 deletions(-) create mode 100644 docs/adr/0021-mpc-bright-asteroid-catalog.md create mode 100644 markers/mrk_asteroid.png create mode 100644 python/PiFinder/asteroid_catalog.py create mode 100644 python/PiFinder/asteroids.py create mode 100644 python/PiFinder/download_utils.py create mode 100644 python/tests/data/asteroids_fixture.txt create mode 100644 python/tests/test_asteroids.py create mode 100644 python/tests/test_config_asteroid_migration.py create mode 100644 python/tests/test_download_utils.py create mode 100644 python/tests/test_dynamic_catalogs.py create mode 100644 python/tests/test_object_list_progress.py create mode 100644 python/tests/test_object_list_sorting.py diff --git a/default_config.json b/default_config.json index 5794660bc..10b35ac90 100644 --- a/default_config.json +++ b/default_config.json @@ -35,6 +35,7 @@ "Str", "PL", "CM", + "MP", "RDS" ], "filter.object_types": [ @@ -52,7 +53,8 @@ "*", "?", "Pla", - "CM" + "CM", + "AS" ], "filter.constellations": [ "And", diff --git a/docs/adr/0021-mpc-bright-asteroid-catalog.md b/docs/adr/0021-mpc-bright-asteroid-catalog.md new file mode 100644 index 000000000..cbcddff10 --- /dev/null +++ b/docs/adr/0021-mpc-bright-asteroid-catalog.md @@ -0,0 +1,52 @@ +# MPC annual bright-minor-planet files for the asteroid catalog + +PiFinder needs an asteroid subset small enough for a Raspberry Pi and useful to +visual observers. The complete MPCORB catalog is orders of magnitude larger +than the set that can become visually observable in a given year. An absolute +magnitude (`H`) cut is also insufficient: it can omit intrinsically faint +near-Earth asteroids during a bright close apparition. + +The `MP` dynamic catalog uses the Minor Planet Center's annual +`Ephemerides/Bright//Soft00Bright.txt` file. It is already curated by +observing year, uses the standard MPCORB one-line format supported by the +pinned Skyfield release, contains numbered asteroids, and supplies the `H` and +`G` photometric parameters. + +## Decisions + +- Load the current year's file and opportunistically merge next year's once it + is published. A previous-year file remains a stale-data fallback around New + Year or during network failure. Duplicate asteroid numbers use the newest + packed element epoch. Because a Pi 4 has no RTC, source-year selection waits + for trustworthy GPS time; before that, the UI may identify an already stored + edition from its filename but never chooses a download year from wall time. +- The minor-planet number is the catalog sequence. Observation logs for virtual + objects are keyed by `(catalog, sequence)`, so `MP 4` must remain Vesta across + refreshes and restarts. +- Propagate the small element set locally in vectorized NumPy operations and + compute apparent magnitude with the IAU H-G phase law. Non-finite values and + objects fainter than the named magnitude-15 catalog safety limit are omitted; + the user's ordinary magnitude and altitude filters remain the observing + controls. +- For visible objects, search 550 days for the first upcoming + ecliptic-longitude opposition (or greatest elongation for an interior object) + and the peak magnitude associated with that apparition. Day zero is excluded + from “upcoming” so an event that just passed is not reported as the next one. +- Catalog source updates are transactional: bytes go to a temporary sibling, + are parsed and validated, and atomically replace the active file. The old + objects and their source edition remain visible during download or after failure. + A populated catalog shows a compact determinate or indeterminate progress bar. +- JPL SBDB is a documented replacement candidate if MPC retires the annual + file, not an automatic fallback. Supporting two unrelated runtime formats + would add failure modes without improving normal operation. +- Existing persisted Catalog and Type filters receive `MP` and `AS` through a + one-time config migration. Its marker ensures a later user choice to disable + asteroids remains authoritative. + +## Consequences + +Asteroid calculations stay bounded to a few hundred source rows, stable numbers +make jump-to and logging meaningful, and close NEO apparitions are retained +without downloading the full MPCORB database. Annual elements are two-body +propagated, so they do not claim Horizons-level precision; vectorized results +are regression-tested against Skyfield's per-object MPCORB orbit builder. diff --git a/docs/ax/catalog.md b/docs/ax/catalog.md index 595afdb99..ec98ddd81 100644 --- a/docs/ax/catalog.md +++ b/docs/ax/catalog.md @@ -31,7 +31,8 @@ At a high level: │ └─► re-filter │ ├── dynamic catalogs: PlanetCatalog (TimerMixin, every ~5 min) - │ CometCatalog (similar) + │ CometCatalog (downloaded MPC elements) + │ AsteroidCatalog (annual MPC bright subset) │ └─► Catalogs object (single instance shared across the app) │ @@ -61,7 +62,9 @@ displays. It's a dataclass that merges three things: - Derived/auxiliary data — `names` (list of strings), `mag` (`MagnitudeObject`), `mag_str` (display string), `logged` (looked up in the observations DB), `last_filtered_time`/`last_filtered_result` - (used by the filter cache). + (used by the filter cache), and optional structured solar-system fields + (`earth_distance_au`, `sun_distance_au`, `opposition_date`, + `peak_magnitude`, `peak_date`). Two `CompositeObject`s are equal iff their `object_id`s match. That means the same underlying object referenced by multiple catalogs (e.g. @@ -161,8 +164,8 @@ menu). It: one `Catalog` per entry in `catalogs_info`, ending up with a `Catalogs` instance even for catalogs that are currently empty (the background loader will populate them later). -5. Appends two dynamic catalogs: `PlanetCatalog` (`PL`) and, via local - import, `CometCatalog`. +5. Appends three dynamic catalogs: `PlanetCatalog` (`PL`), `CometCatalog` + (`CM`), and `AsteroidCatalog` (`MP`). 6. Asserts `check_catalogs_sequences(...)`. The reference to the background loader is also stashed on the @@ -223,9 +226,14 @@ The filter therefore caches per-object decisions: the most recent decision for that object. - `apply_filter(obj)` short-circuits if `obj.last_filtered_time > self.dirty_time` — i.e. the object was filtered after the last change. +Runtime position/catalog replacements call `invalidate_filter_cache()` for +the changed catalog and set a catalog-content-dirty flag. This wakes an open +list without advancing `dirty_time`, so unchanged catalogs retain their cached +results. Automatic list rebuilds reapply the current sort silently, reserving +the “Sorting by…” toast for a user-selected sort. -So if the filter has not changed since the last sweep, the second sweep -is O(n) cache reads with no real predicate work. +So if the filter has not changed since the last sweep, the next sweep is +served from cached verdicts without recomputing the predicates. ### 4.2 Altitude requires GPS @@ -307,10 +315,28 @@ A `Catalog` subclass that: ### 6.2 `CometCatalog` Imported locally in `CatalogBuilder.build()` to avoid a circular -import. Same general pattern: dynamic, status-aware, registered as a -regular `Catalog` in the `Catalogs` collection. +import. It downloads MPC comet elements transactionally and keeps the active +objects visible while a replacement downloads. `CatalogStatus` carries known +percentage progress or `None` for an indeterminate progress bar. Successful +downloads trigger a full recalculation; failed downloads leave both the old +objects and their displayed source age intact. -### 6.3 `TimerMixin` +### 6.3 `AsteroidCatalog` + +The `MP` catalog reads MPC's annual `Soft00Bright.txt` observing subset (ADR +0021). Its catalog listing sequence is the stable numbered minor planet, while +its negative object ID remains session-minted like every virtual object. +Vectorized propagation supplies J2000 RA/Dec, Earth/Sun distance, and an IAU +H-G apparent magnitude. Objects above the magnitude-15 safety limit are omitted; +ordinary catalog filters can impose a brighter user limit. + +Visible objects are enriched across the next 550 days with the first upcoming +ecliptic-longitude opposition (or greatest elongation) plus the peak magnitude +and date for that apparition. These structured values drive the Asteroids list's +Brightest, Distance, and Opposition sorts; description text is presentation +only. + +### 6.4 `TimerMixin` Provides `start_timer()` / `stop()` plus a `time_delay_seconds` that can be either an int or a callable. Each fire schedules itself again via @@ -319,7 +345,7 @@ catalog method does not run on the timer thread directly — useful because `do_timed_task` can take a noticeable amount of time (`sf_utils.calc_planets` is not cheap). -### 6.4 `VirtualIDManager` +### 6.5 `VirtualIDManager` Static helper that hands out monotonically decreasing `object_id` values for non-DB objects. Held under `virtual_id_lock` and persists diff --git a/docs/ax/catalog/CONTEXT.md b/docs/ax/catalog/CONTEXT.md index ec25401b4..435cfa726 100644 --- a/docs/ax/catalog/CONTEXT.md +++ b/docs/ax/catalog/CONTEXT.md @@ -9,7 +9,7 @@ The Catalog context owns runtime loading, filtering, searching and display of as ### Identity **Catalog code**: -Short string identifier for a catalog as a whole — `"M"`, `"NGC"`, `"IC"`, `"WDS"`, `"PL"` (planets). Drives DB queries and the UI designator. Its readable sibling is the **catalog display name** ("Collinder" for code `"Cr"`). +Short string identifier for a catalog as a whole — `"M"`, `"NGC"`, `"IC"`, `"WDS"`, `"PL"` (planets), `"MP"` (asteroids). Drives DB queries and the UI designator. Its readable sibling is the **catalog display name** ("Collinder" for code `"Cr"`). _Avoid_: catalog id, prefix; for the readable form say **catalog display name**, not "catalog name". **Catalog display name**: @@ -98,6 +98,10 @@ _Avoid_: logged (when speaking to users or in UI copy), seen. Pair of epoch timestamps that drive the per-object filter cache. An object is re-evaluated only when `obj.last_filtered_time < filter.dirty_time`. _Avoid_: invalidation, cache key. +**Catalog content dirty**: +A wake-up flag for runtime object changes under unchanged filter criteria. The changed catalog resets its own cached verdicts, then `mark_catalog_content_dirty()` makes open lists rebuild without advancing **Dirty time**. +_Avoid_: filter dirty (the criteria did not change), stale (time did not expire the verdict). + **Empty-list rejection**: For `object_types` and `constellations`, an empty list rejects every object; only `None` (or `"Any"` for `observed`) means "don't filter on this dimension." Flagged here because it surprises callers. _Avoid_: empty filter, no filter. @@ -131,7 +135,7 @@ _Avoid_: load-done event, ready signal. ### Dynamic catalogs **Dynamic catalog**: -A `Catalog` whose objects are computed at runtime rather than loaded from `objects.db`. Currently `PlanetCatalog` and `CometCatalog`. +A `Catalog` whose objects are computed at runtime rather than loaded from `objects.db`. Currently `PlanetCatalog`, `CometCatalog`, and `AsteroidCatalog`. _Avoid_: live catalog, computed catalog. **PlanetCatalog**: @@ -142,6 +146,18 @@ _Avoid_: planets, ephemeris (those are more general). Comet equivalent of `PlanetCatalog`. Imported locally inside `CatalogBuilder.build()` to break a circular import. _Avoid_: comets. +**AsteroidCatalog**: +The `"MP"` dynamic catalog. Loads the MPC annual bright-minor-planet subset, uses the numbered minor planet as its stable **Sequence**, computes current position and H-G apparent magnitude, and enriches visible objects with distance, opposition/greatest-elongation, and apparition-peak data. Source replacement is atomic; populated objects remain available during download. +_Avoid_: minor planets (when the concrete catalog class is meant), `Ast` (that code means asterism). + +**Catalog data age**: +Whole days since the active downloaded elements file's server timestamp. Used for frequently refreshed sources such as comets. During refresh this continues to describe the objects actually on screen; it changes only after a validated file replaces the active source. Annual asteroid data instead shows its compact MPC edition label, because a large day count is normal rather than a stale-data warning. +_Avoid_: catalog age (ambiguous with the runtime `Catalog` object). + +**Download progress**: +The `CatalogStatus.data["progress"]` percentage for a downloaded dynamic catalog. `None` means the response has no known Content-Length and the UI draws an indeterminate bar. Progress does not imply the old objects have been removed; they stay usable until recalculation succeeds. +_Avoid_: loading progress (deferred database catalog loading is a different lifecycle). + **TimerMixin**: Composition helper providing self-rescheduling periodic timers. `time_delay_seconds` may be a callable for adaptive delays. Updates run in their own thread, not on the timer thread. _Avoid_: scheduler, ticker. diff --git a/docs/ax/catalog/obslist-formats/README.md b/docs/ax/catalog/obslist-formats/README.md index 499340858..f4470ec88 100644 --- a/docs/ax/catalog/obslist-formats/README.md +++ b/docs/ax/catalog/obslist-formats/README.md @@ -133,6 +133,7 @@ like `"Galaxy"`. The canonical set is defined by `OBJ_TYPES` in | `PN` | Planetary nebula | | `Kt` | Knot | | `DN` | Dark nebula | | `Pla` | Planet | | `C+N` | Cluster + nebula | | `CM` | Comet | +| `AS` | Asteroid | | | | | | | | `?` | Unknown | A code drives two things: the symbol drawn next to the object, and the **Type** diff --git a/markers/mrk_asteroid.png b/markers/mrk_asteroid.png new file mode 100644 index 0000000000000000000000000000000000000000..37ccc4805ab1fa4790c696b3601e705f7dda4965 GIT binary patch literal 126 zcmeAS@N?(olHy`uVBq!ia0vp^+#t-s1SHkYJtzcHuAVNAAr*6K{T=xj6gVbb`@i3( zU1sLA7!gs)4GfcZGbV7VY+bkILp6tfLrluUO{@IoAIn#KS`+s~Cvf5~y>lmp+~)>1 Zv!3nqJ-yb7WgE~y22WQ%mvv4FO#r`?Dl7m1 literal 0 HcmV?d00001 diff --git a/python/PiFinder/asteroid_catalog.py b/python/PiFinder/asteroid_catalog.py new file mode 100644 index 000000000..11f613074 --- /dev/null +++ b/python/PiFinder/asteroid_catalog.py @@ -0,0 +1,291 @@ +"""Dynamic catalog of MPC's bright asteroids for the observing year.""" + +from __future__ import annotations + +import datetime +import logging +import threading +from pathlib import Path +from typing import Optional + +import pytz + +import PiFinder.asteroids as asteroids +from PiFinder import timez +from PiFinder.calc_utils import sf_utils +from PiFinder.catalog_base import ( + CatalogState, + CatalogStatus, + TimerMixin, + VirtualIDManager, +) +from PiFinder.catalogs import Catalog +from PiFinder.composite_object import CompositeObject, MagnitudeObject, SizeObject +from PiFinder.state import SharedStateObj +from PiFinder.utils import Timer, asteroid_data_dir + + +logger = logging.getLogger("AsteroidCatalog") + + +class AsteroidCatalog(Catalog): + POSITION_UPDATE_SECONDS = 601 + WAITING_FOR_DATA_SECONDS = 10 + + def __init__( + self, + dt: datetime.datetime, + shared_state: SharedStateObj, + data_directory: Path = asteroid_data_dir, + ): + self._timer = TimerMixin() + self._virtual_id_manager = VirtualIDManager() + super().__init__("MP", "Asteroids") + self.shared_state = shared_state + self.data_directory = data_directory + self._task_lock = threading.Lock() + self._download_lock = threading.Lock() + self.download_progress: Optional[int] = None + self.calculation_progress: Optional[int] = None + self._is_downloading = False + self._cached_file_mtime: Optional[float] = None + self._last_state = CatalogState.READY + self.initialized = False + + self._timer.do_timed_task = self.do_timed_task + self._timer.time_delay_seconds = lambda: self.time_delay_seconds + + if self.shared_state.altaz_ready() and self._element_files(dt): + self.do_timed_task() + threading.Thread(target=self._refresh_sources, daemon=True).start() + self._timer.start_timer() + self._start_background_retry() + + def _element_files(self, dt: datetime.datetime) -> list[Path]: + return asteroids.available_element_files(dt, self.data_directory) + + @property + def time_delay_seconds(self) -> int: + return ( + self.POSITION_UPDATE_SECONDS + if self.initialized + else self.WAITING_FOR_DATA_SECONDS + ) + + def get_age(self) -> Optional[int]: + if not self.shared_state.altaz_ready(): + return None + files = self._element_files(self.shared_state.datetime()) + if not files: + return None + newest_mtime = max(path.stat().st_mtime for path in files) + self._cached_file_mtime = newest_mtime + local_date = timez.utc_from_timestamp(newest_mtime) + now = self.shared_state.datetime() + if now.tzinfo is None: + now = pytz.UTC.localize(now) + return round((now - local_date).total_seconds() / 86400.0) + + def get_data_label(self) -> Optional[str]: + """Annual MPC sets are editions, not feeds that become stale daily.""" + dt = self.shared_state.datetime() + if dt is not None: + current = asteroids.asteroid_file_for_year(dt.year, self.data_directory) + edition_year = dt.year if current.exists() else dt.year - 1 + return f"MPC {edition_year}" + + # A Pi 4 has no RTC, so its wall clock is not trustworthy before GPS. + # Report only an edition that is proven by an on-disk filename. + years = [] + for path in self.data_directory.glob("Soft00Bright-*.txt"): + try: + years.append(int(path.stem.rsplit("-", 1)[1])) + except (IndexError, ValueError): + continue + return f"MPC {max(years)}" if years else None + + def get_status(self) -> CatalogStatus: + if self._is_downloading: + current = CatalogState.DOWNLOADING + elif not self.shared_state.altaz_ready(): + current = CatalogState.NO_GPS + elif self.calculation_progress is not None or not self.initialized: + current = CatalogState.CALCULATING + else: + current = CatalogState.READY + data = None + if current == CatalogState.DOWNLOADING: + data = {"progress": self.download_progress} + elif ( + current == CatalogState.CALCULATING + and self.calculation_progress is not None + ): + data = {"progress": self.calculation_progress} + status = CatalogStatus(current, self._last_state, data) + self._last_state = current + return status + + def _download_year(self, year: int) -> bool: + if not self._download_lock.acquire(blocking=False): + return False + try: + self._is_downloading = True + self.download_progress = 0 + + def progress(value: Optional[int]) -> None: + self.download_progress = value + + result = asteroids.download_asteroid_year( + year, self.data_directory, progress_callback=progress + ) + if result.success: + self._cached_file_mtime = result.file_mtime + return result.success + finally: + self._is_downloading = False + self.download_progress = None + self._download_lock.release() + + def _refresh_sources(self, force_recalculate: bool = False) -> None: + if not self.shared_state.altaz_ready(): + logger.info("Deferring asteroid source selection until GPS time is ready") + return + dt = self.shared_state.datetime() + if dt is None: + return + changed = False + # Current year is required. If MPC has not published it yet, fetch the + # previous year as an explicitly stale New-Year fallback. Next year is + # opportunistic; a 404 leaves all active data untouched. + for year in (dt.year,): + needed, reason = asteroids.check_asteroid_download_needed( + year, self.data_directory + ) + if needed: + logger.info("Asteroid data %s: %s", year, reason) + changed = self._download_year(year) or changed + if not asteroids.asteroid_file_for_year(dt.year, self.data_directory).exists(): + previous_year = dt.year - 1 + needed, reason = asteroids.check_asteroid_download_needed( + previous_year, self.data_directory + ) + if needed: + logger.info("Asteroid fallback data %s: %s", previous_year, reason) + changed = self._download_year(previous_year) or changed + + next_year = dt.year + 1 + needed, reason = asteroids.check_asteroid_download_needed( + next_year, self.data_directory + ) + if needed: + logger.info("Asteroid next-year data %s: %s", next_year, reason) + changed = self._download_year(next_year) or changed + if (changed or force_recalculate) and self.shared_state.altaz_ready(): + self.do_timed_task() + + def refresh(self) -> None: + threading.Thread( + target=self._refresh_sources, + kwargs={"force_recalculate": True}, + daemon=True, + ).start() + + def _start_background_retry(self) -> None: + def retry() -> None: + retry_wait = threading.Event() + while True: + if not self.shared_state.altaz_ready(): + retry_wait.wait(self.WAITING_FOR_DATA_SECONDS) + continue + dt = self.shared_state.datetime() + if dt is None: + retry_wait.wait(self.WAITING_FOR_DATA_SECONDS) + continue + if self._element_files(dt): + break + self._refresh_sources() + if self._element_files(dt): + break + retry_wait.wait(60) + + threading.Thread(target=retry, daemon=True).start() + + def _make_object(self, asteroid: dict) -> CompositeObject: + ra, dec = asteroid["radec"] + mag = MagnitudeObject([asteroid["mag"]]) + opposition_kind = asteroid.get("opposition_kind", "Opposition") + opposition_date = asteroid.get("opposition_date") + peak_date = asteroid.get("peak_date") + description_lines = [] + if opposition_date: + event_label = "Opp" if opposition_kind == "Opposition" else "Elong" + description_lines.append(f"{event_label}: {opposition_date.isoformat()}") + else: + description_lines.append("Opp: unavailable") + if peak_date: + description_lines.append( + f"Peak {asteroid['peak_magnitude']:.1f}: {peak_date.isoformat()}" + ) + description_lines.extend( + ( + f"Earth: {asteroid['earth_distance']:.2f} AU", + f"Sun: {asteroid['sun_distance']:.2f} AU", + f"Motion: {asteroid['angular_motion_arcsec_per_hour']:.1f}\"/h", + ) + ) + description = "\n".join(description_lines) + return CompositeObject.from_dict( + { + "id": -1, + "obj_type": "AS", + "ra": ra, + "dec": dec, + "const": sf_utils.radec_to_constellation(ra, dec), + "size": SizeObject([]), + "mag": mag, + "mag_str": mag.calc_two_mag_representation(), + "names": [asteroid["name"]], + "catalog_code": "MP", + "sequence": asteroid["number"], + "description": description, + "earth_distance_au": asteroid["earth_distance"], + "sun_distance_au": asteroid["sun_distance"], + "angular_motion_arcsec_per_hour": asteroid[ + "angular_motion_arcsec_per_hour" + ], + "opposition_date": opposition_date, + "opposition_kind": opposition_kind, + "peak_magnitude": asteroid.get("peak_magnitude"), + "peak_date": peak_date, + } + ) + + def init_asteroids(self, dt: datetime.datetime) -> None: + self.calculation_progress = 0 + + def progress(value: int) -> None: + self.calculation_progress = value + + calculated = asteroids.calc_asteroids( + dt, self._element_files(dt), progress_callback=progress + ) + if not calculated: + self.initialized = bool(self.get_objects()) + self.calculation_progress = None + return + objects = [self._make_object(item) for item in calculated.values()] + self.replace_objects(objects) + self._virtual_id_manager.mint_ids(self) + if self.catalog_filter is not None: + self.catalog_filter.mark_catalog_content_dirty() + self.initialized = True + self.calculation_progress = None + + def do_timed_task(self) -> None: + with self._task_lock: + with Timer("Asteroid Catalog periodic update"): + if not self.shared_state.altaz_ready(): + return + dt = self.shared_state.datetime() + if self._element_files(dt): + self.init_asteroids(dt) diff --git a/python/PiFinder/asteroids.py b/python/PiFinder/asteroids.py new file mode 100644 index 000000000..826dfc4ed --- /dev/null +++ b/python/PiFinder/asteroids.py @@ -0,0 +1,414 @@ +"""MPC bright-asteroid elements, propagation, photometry, and apparitions.""" + +from __future__ import annotations + +import logging +import math +import re +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Optional + +import numpy as np +import pandas as pd +from skyfield.data import mpc +from skyfield.data.spice import inertial_frames +from skyfield.timelib import julian_day + +from PiFinder.calc_utils import sf_utils +from PiFinder.download_utils import ( + DownloadResult, + check_download_needed, + download_atomic, +) +from PiFinder.utils import Timer, asteroid_data_dir + + +logger = logging.getLogger("Asteroids") + +MPC_BRIGHT_URL = ( + "https://minorplanetcenter.net/iau/Ephemerides/Bright/{year}/" "Soft00Bright.txt" +) +ASTEROID_VISIBLE_MAG_LIMIT = 15.0 +APPARITION_SEARCH_DAYS = 550 +OPPOSITION_MIN_ELONGATION_DEG = 170.0 +_NUMBER_RE = re.compile(r"^\((\d+)\)(?:\s+(.*))?$") +_ECLIPTIC_TO_ICRF = inertial_frames["ECLIPJ2000"].T + + +def asteroid_file_for_year(year: int, directory: Path = asteroid_data_dir) -> Path: + return directory / f"Soft00Bright-{year}.txt" + + +def asteroid_url_for_year(year: int) -> str: + return MPC_BRIGHT_URL.format(year=year) + + +def _validate_asteroid_file(path: Path) -> None: + with path.open("rb") as source: + dataframe = mpc.load_mpcorb_dataframe(source) + if dataframe.empty: + raise ValueError("MPC bright-asteroid file contains no objects") + if ( + dataframe["designation"] + .map(lambda value: bool(_NUMBER_RE.match(str(value)))) + .sum() + == 0 + ): + raise ValueError("MPC bright-asteroid file has no numbered asteroids") + + +def download_asteroid_year( + year: int, + directory: Path = asteroid_data_dir, + progress_callback: Optional[Callable[[Optional[int]], None]] = None, +) -> DownloadResult: + return download_atomic( + asteroid_url_for_year(year), + asteroid_file_for_year(year, directory), + progress_callback=progress_callback, + validator=_validate_asteroid_file, + ) + + +def check_asteroid_download_needed( + year: int, directory: Path = asteroid_data_dir +) -> tuple[bool, str]: + return check_download_needed( + asteroid_file_for_year(year, directory), asteroid_url_for_year(year) + ) + + +def available_element_files( + dt: datetime, directory: Path = asteroid_data_dir +) -> list[Path]: + """Newest useful annual files, including next year when MPC has published it.""" + years = (dt.year - 1, dt.year, dt.year + 1) + return [ + asteroid_file_for_year(year, directory) + for year in years + if asteroid_file_for_year(year, directory).exists() + ] + + +def load_asteroids_dataframe(paths: list[Path]) -> pd.DataFrame: + frames = [] + for path in paths: + with path.open("rb") as source: + frames.append(mpc.load_mpcorb_dataframe(source)) + if not frames: + return pd.DataFrame() + dataframe = pd.concat(frames, ignore_index=True) + numeric = ( + "magnitude_H", + "magnitude_G", + "mean_anomaly_degrees", + "argument_of_perihelion_degrees", + "longitude_of_ascending_node_degrees", + "inclination_degrees", + "eccentricity", + "mean_daily_motion_degrees", + "semimajor_axis_au", + ) + for column in numeric: + dataframe[column] = pd.to_numeric(dataframe[column], errors="coerce") + dataframe["magnitude_G"] = dataframe["magnitude_G"].fillna(0.15) + required = [column for column in numeric if column != "magnitude_G"] + [ + "epoch_packed", + "designation", + ] + dataframe = dataframe.dropna(subset=required) + dataframe = dataframe[ + (dataframe.eccentricity >= 0.0) & (dataframe.eccentricity < 1.0) + ] + dataframe["number"] = dataframe.designation.map(_minor_planet_number) + dataframe = dataframe.dropna(subset=["number"]) + dataframe["number"] = dataframe.number.astype(int) + # Lexical order of the MPC packed epoch is chronological within these + # modern annual files. Prefer the freshest duplicate across year files. + return ( + dataframe.sort_values("epoch_packed") + .drop_duplicates(subset=["number"], keep="last") + .sort_values("number") + .reset_index(drop=True) + ) + + +def _minor_planet_number(designation: str) -> Optional[int]: + match = _NUMBER_RE.match(str(designation).strip()) + return int(match.group(1)) if match else None + + +def minor_planet_name(designation: str) -> str: + match = _NUMBER_RE.match(str(designation).strip()) + if not match: + return str(designation).strip() + return (match.group(2) or match.group(1)).strip() + + +def _packed_epoch_jd(value: str) -> float: + def unpack(char: str) -> int: + return ord(char) - (48 if char.isdigit() else 55) + + value = str(value) + year = 100 * unpack(value[0]) + int(value[1:3]) + return julian_day(year, unpack(value[3]), unpack(value[4])) - 0.5 + + +def _heliocentric_positions(dataframe: pd.DataFrame, tt_jd) -> np.ndarray: + """Return heliocentric ICRF positions shaped ``(3, objects, times)``.""" + target_jd = np.atleast_1d(np.asarray(tt_jd, dtype=float)) + epoch_jd = np.asarray([_packed_epoch_jd(v) for v in dataframe.epoch_packed]) + mean_anomaly = np.radians(dataframe.mean_anomaly_degrees.to_numpy(float))[:, None] + mean_motion = np.radians(dataframe.mean_daily_motion_degrees.to_numpy(float))[ + :, None + ] + anomaly = ( + mean_anomaly + mean_motion * (target_jd[None, :] - epoch_jd[:, None]) + ) % (2.0 * math.pi) + eccentricity = dataframe.eccentricity.to_numpy(float)[:, None] + + eccentric_anomaly = anomaly.copy() + for _ in range(12): + correction = ( + eccentric_anomaly - eccentricity * np.sin(eccentric_anomaly) - anomaly + ) / (1.0 - eccentricity * np.cos(eccentric_anomaly)) + eccentric_anomaly -= correction + if np.max(np.abs(correction)) < 1e-13: + break + + semimajor = dataframe.semimajor_axis_au.to_numpy(float)[:, None] + x_orbit = semimajor * (np.cos(eccentric_anomaly) - eccentricity) + y_orbit = semimajor * np.sqrt(1.0 - eccentricity**2) * np.sin(eccentric_anomaly) + + node = np.radians(dataframe.longitude_of_ascending_node_degrees.to_numpy(float))[ + :, None + ] + peri = np.radians(dataframe.argument_of_perihelion_degrees.to_numpy(float))[:, None] + inc = np.radians(dataframe.inclination_degrees.to_numpy(float))[:, None] + cos_node, sin_node = np.cos(node), np.sin(node) + cos_peri, sin_peri = np.cos(peri), np.sin(peri) + cos_inc, sin_inc = np.cos(inc), np.sin(inc) + + x = (cos_node * cos_peri - sin_node * sin_peri * cos_inc) * x_orbit + ( + -cos_node * sin_peri - sin_node * cos_peri * cos_inc + ) * y_orbit + y = (sin_node * cos_peri + cos_node * sin_peri * cos_inc) * x_orbit + ( + -sin_node * sin_peri + cos_node * cos_peri * cos_inc + ) * y_orbit + z = sin_peri * sin_inc * x_orbit + cos_peri * sin_inc * y_orbit + return np.einsum("ij,jnt->int", _ECLIPTIC_TO_ICRF, np.array([x, y, z])) + + +def hg_magnitude( + magnitude_h, + magnitude_g, + sun_distance, + observer_distance, + phase_angle_radians, +): + """IAU H-G apparent visual magnitude model.""" + tan_half = np.tan(np.clip(phase_angle_radians, 0.0, math.pi - 1e-9) / 2.0) + phi1 = np.exp(-3.33 * np.power(tan_half, 0.63)) + phi2 = np.exp(-1.87 * np.power(tan_half, 1.22)) + phase = (1.0 - magnitude_g) * phi1 + magnitude_g * phi2 + return ( + magnitude_h + + 5.0 * np.log10(sun_distance * observer_distance) + - 2.5 * np.log10(phase) + ) + + +def _geometry(dataframe: pd.DataFrame, times, observer_positions: np.ndarray): + helio = _heliocentric_positions(dataframe, times.tt) + sun = sf_utils.eph["sun"].at(times).position.au + if sun.ndim == 1: + sun = sun[:, None] + observer = observer_positions + if observer.ndim == 1: + observer = observer[:, None] + topocentric = sun[:, None, :] + helio - observer[:, None, :] + earth_distance = np.linalg.norm(topocentric, axis=0) + sun_distance = np.linalg.norm(helio, axis=0) + asteroid_to_sun = -helio + asteroid_to_observer = -topocentric + cos_phase = np.sum(asteroid_to_sun * asteroid_to_observer, axis=0) / ( + sun_distance * earth_distance + ) + phase_angle = np.arccos(np.clip(cos_phase, -1.0, 1.0)) + h = dataframe.magnitude_H.to_numpy(float)[:, None] + g = dataframe.magnitude_G.to_numpy(float)[:, None] + magnitude = hg_magnitude(h, g, sun_distance, earth_distance, phase_angle) + return helio, topocentric, earth_distance, sun_distance, magnitude + + +def _radec(topocentric: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + eq_pos = np.einsum("ij,jnt->int", sf_utils.ts.J2000.M, topocentric) + distance = np.linalg.norm(eq_pos, axis=0) + ra = np.degrees(np.arctan2(eq_pos[1], eq_pos[0])) % 360.0 + dec = np.degrees(np.arcsin(np.clip(eq_pos[2] / distance, -1.0, 1.0))) + return ra, dec + + +def _angular_motion_arcsec_per_hour(topocentric: np.ndarray) -> np.ndarray: + """Return apparent one-hour sky motion for each object.""" + unit = topocentric / np.linalg.norm(topocentric, axis=0)[None, :, :] + cos_separation = np.sum(unit[:, :, 0] * unit[:, :, 1], axis=0) + return np.degrees(np.arccos(np.clip(cos_separation, -1.0, 1.0))) * 3600.0 + + +def _next_apparition_index(separation: np.ndarray) -> tuple[int, bool]: + """Return the first future local maximum, never the day-0 endpoint.""" + local_maxima = ( + np.nonzero( + (separation[1:-1] >= separation[:-2]) & (separation[1:-1] >= separation[2:]) + )[0] + + 1 + ) + opposition_candidates = local_maxima[ + separation[local_maxima] >= OPPOSITION_MIN_ELONGATION_DEG + ] + if len(opposition_candidates): + return int(opposition_candidates[0]), True + if len(local_maxima): + return int(local_maxima[0]), False + + # A maximum can fall on the far scan boundary. Day 0 remains excluded: it + # might be an event that passed minutes ago and is therefore not "next". + future_index = int(np.nanargmax(separation[1:])) + 1 + return ( + future_index, + bool(separation[future_index] >= OPPOSITION_MIN_ELONGATION_DEG), + ) + + +def _apparitions(dataframe: pd.DataFrame, dt: datetime) -> dict[int, dict[str, Any]]: + start = sf_utils.ts.from_datetime(dt) + times = sf_utils.ts.tt_jd(start.tt + np.arange(APPARITION_SEARCH_DAYS + 1)) + earth = sf_utils.earth.at(times).position.au + _helio, geocentric, earth_distance, _sun_distance, magnitude = _geometry( + dataframe, times, earth + ) + sun = sf_utils.eph["sun"].at(times).position.au + sun_from_earth = sun - earth + cos_elongation = np.sum(geocentric * sun_from_earth[:, None, :], axis=0) / ( + earth_distance * np.linalg.norm(sun_from_earth, axis=0)[None, :] + ) + elongation = np.degrees(np.arccos(np.clip(cos_elongation, -1.0, 1.0))) + # Opposition is a 180-degree *ecliptic-longitude* separation. Its true + # angular elongation can be noticeably smaller for high-latitude objects. + to_ecliptic = _ECLIPTIC_TO_ICRF.T + asteroid_ecliptic = np.einsum("ij,jnt->int", to_ecliptic, geocentric) + sun_ecliptic = to_ecliptic @ sun_from_earth + asteroid_lon = np.degrees(np.arctan2(asteroid_ecliptic[1], asteroid_ecliptic[0])) + sun_lon = np.degrees(np.arctan2(sun_ecliptic[1], sun_ecliptic[0]))[None, :] + longitude_separation = np.abs((asteroid_lon - sun_lon + 180.0) % 360.0 - 180.0) + + result: dict[int, dict[str, Any]] = {} + datetimes = times.utc_datetime() + for row_index, number in enumerate(dataframe.number.to_numpy(int)): + separation = longitude_separation[row_index] + opposition_index, is_opposition = _next_apparition_index(separation) + # Keep peak brightness tied to this apparition instead of selecting a + # second, brighter opposition near the far edge of the 18-month scan. + peak_start = max(0, opposition_index - 90) + peak_stop = min(len(times), opposition_index + 91) + peak_index = peak_start + int( + np.nanargmin(magnitude[row_index, peak_start:peak_stop]) + ) + maximum_elongation = float(elongation[row_index, opposition_index]) + result[number] = { + "opposition_date": datetimes[opposition_index].date(), + "opposition_kind": "Opposition" if is_opposition else "Greatest elongation", + "maximum_elongation_deg": maximum_elongation, + "peak_date": datetimes[peak_index].date(), + "peak_magnitude": float(magnitude[row_index, peak_index]), + } + return result + + +def process_asteroid(row, dt: datetime) -> dict[str, Any]: + dataframe = pd.DataFrame([row]) + result = _calculate_dataframe(dataframe, dt, include_apparitions=False) + return next(iter(result.values()), {}) + + +def _calculate_dataframe( + dataframe: pd.DataFrame, + dt: datetime, + include_apparitions: bool = True, +) -> dict[int, dict[str, Any]]: + if dataframe.empty: + return {} + time = sf_utils.ts.from_datetime(dt) + motion_times = sf_utils.ts.tt_jd(np.array([time.tt, time.tt + 1.0 / 24.0])) + observer = sf_utils.observer_loc.at(motion_times).position.au + _, topocentric, earth_distance, sun_distance, magnitude = _geometry( + dataframe, motion_times, observer + ) + angular_motion = _angular_motion_arcsec_per_hour(topocentric) + ra, dec = _radec(topocentric) + visible = np.isfinite(magnitude[:, 0]) & ( + magnitude[:, 0] <= ASTEROID_VISIBLE_MAG_LIMIT + ) + visible_df = dataframe.loc[visible].reset_index(drop=True) + apparitions = ( + _apparitions(visible_df, dt) if include_apparitions and len(visible_df) else {} + ) + + result: dict[int, dict[str, Any]] = {} + for source_index in np.nonzero(visible)[0]: + row = dataframe.iloc[source_index] + number = int(row.number) + item = { + "number": number, + "name": minor_planet_name(row.designation), + "full_name": str(row.designation).strip(), + "radec": (float(ra[source_index, 0]), float(dec[source_index, 0])), + "mag": float(magnitude[source_index, 0]), + "earth_distance": float(earth_distance[source_index, 0]), + "sun_distance": float(sun_distance[source_index, 0]), + "angular_motion_arcsec_per_hour": float(angular_motion[source_index]), + } + item.update(apparitions.get(number, {})) + result[number] = item + return result + + +def calc_asteroids( + dt: datetime, + paths: Optional[list[Path]] = None, + progress_callback: Optional[Callable[[int], None]] = None, +) -> dict[int, dict[str, Any]]: + with Timer("calc_asteroids()"): + if sf_utils.observer_loc is None or dt is None: + return {} + if progress_callback: + progress_callback(0) + dataframe = load_asteroids_dataframe(paths or available_element_files(dt)) + if progress_callback: + progress_callback(10) + if dataframe.empty: + return {} + try: + result = _calculate_dataframe(dataframe, dt) + except Exception: + logger.error( + "VECTORIZED ASTEROID PROPAGATION FAILED — using per-object fallback", + exc_info=True, + ) + result = {} + total = len(dataframe) + for index, (_, row) in enumerate(dataframe.iterrows(), 1): + try: + item = process_asteroid(row, dt) + except Exception as exc: + logger.warning("Skipping asteroid %s: %s", row.designation, exc) + continue + if item: + result[int(item["number"])] = item + if progress_callback: + progress_callback(10 + int(90 * index / total)) + if progress_callback: + progress_callback(100) + return result diff --git a/python/PiFinder/catalog_base.py b/python/PiFinder/catalog_base.py index ac103cedb..f2bd1771a 100644 --- a/python/PiFinder/catalog_base.py +++ b/python/PiFinder/catalog_base.py @@ -120,6 +120,17 @@ def add_objects(self, objects: List): assert self.check_sequences() self.last_filtered = 0 # objects changed -> invalidate filter cache + def replace_objects(self, objects: List) -> None: + """Replace a dynamic catalog in one assignment, then rebuild indices.""" + replacement = list(objects) + replacement.sort(key=self.sort) + assert len({obj.sequence for obj in replacement}) == len(replacement) + self.__objects = replacement + self.max_sequence = max((obj.sequence for obj in replacement), default=0) + self._update_id_to_pos() + self._update_sequence_to_pos() + self.last_filtered = 0 + def clear_objects(self): """ Remove all objects and reset the sequence/id indexes. diff --git a/python/PiFinder/catalogs.py b/python/PiFinder/catalogs.py index e66a458e4..bea04716e 100644 --- a/python/PiFinder/catalogs.py +++ b/python/PiFinder/catalogs.py @@ -116,6 +116,10 @@ def __init__( self._constellations = constellations self._selected_catalogs = set(selected_catalogs) self.last_filtered_time = 0 + # Dynamic catalogs can replace their objects without changing the + # active filter criteria. Wake open lists without invalidating every + # unchanged catalog's cached result. + self._catalog_content_dirty = False def load_from_config(self, config_object: Config): """ @@ -128,11 +132,18 @@ def load_from_config(self, config_object: Config): self._constellations = config_object.get_option("filter.constellations", []) self._selected_catalogs = config_object.get_option("filter.selected_catalogs") self.last_filtered_time = 0 + self._catalog_content_dirty = False def mark_dirty(self): """Mark the filter as dirty, triggering a re-filter on next check""" self.dirty_time = time.time() + def mark_catalog_content_dirty(self) -> None: + self._catalog_content_dirty = True + + def clear_catalog_content_dirty(self) -> None: + self._catalog_content_dirty = False + @property def magnitude(self): return self._magnitude @@ -206,6 +217,8 @@ def is_dirty(self) -> bool: Returns true if the filter parameters have changed since the last filter. False if not """ + if self._catalog_content_dirty: + return True if self.last_filtered_time > self.dirty_time: return False else: @@ -339,6 +352,12 @@ def filter_objects(self) -> List[CompositeObject]: self.last_filtered = time.time() return self.filtered_objects + def invalidate_filter_cache(self) -> None: + """Invalidate only this catalog after its runtime objects change.""" + self.last_filtered = 0 + for obj in self._get_objects(): + obj.last_filtered_time = 0 + def get_filtered_objects(self): return self.filtered_objects @@ -349,6 +368,10 @@ def get_age(self) -> Optional[int]: """If the catalog data is time-sensitive, return age in days.""" return None + def get_data_label(self) -> Optional[str]: + """Optional compact source-edition label for object-list headers.""" + return None + def get_status(self) -> CatalogStatus: """ Return the current status of the catalog with transition tracking. @@ -384,6 +407,8 @@ def filter_catalogs(self): """ for catalog in self.__catalogs: catalog.filter_objects() + if self.catalog_filter is not None: + self.catalog_filter.clear_catalog_content_dirty() def set_catalog_filter(self, catalog_filter: CatalogFilter) -> None: """ @@ -631,7 +656,6 @@ def init_planets(self, dt): if not planet_dict: logger.debug("No GPS lock during initialization - will retry soon") - self.initialised = True # Still mark as initialized so timer starts return sequence = 0 @@ -677,6 +701,7 @@ def do_timed_task(self): dt = self.shared_state.datetime() if not self.initialized: self.init_planets(dt) + return planet_dict = sf_utils.calc_planets(dt) @@ -698,6 +723,9 @@ def do_timed_task(self): obj.mag_str = obj.mag.calc_two_mag_representation() except (KeyError, ValueError) as e: logger.error(f"Error updating planet {name}: {e}") + self.invalidate_filter_cache() + if self.catalog_filter is not None: + self.catalog_filter.mark_catalog_content_dirty() class CatalogBackgroundLoader: @@ -940,6 +968,14 @@ def build(self, shared_state, ui_queue=None) -> Catalogs: ) all_catalogs.add(comet_catalog) + from PiFinder.asteroid_catalog import AsteroidCatalog + + asteroid_catalog: Catalog = AsteroidCatalog( + timez.utc_now(), + shared_state=shared_state, + ) + all_catalogs.add(asteroid_catalog) + assert self.check_catalogs_sequences(all_catalogs) is True return all_catalogs @@ -949,7 +985,7 @@ def check_catalogs_sequences(self, catalogs: Catalogs): if not result: logger.error("Duplicate sequence catalog %s!", catalog.catalog_code) return False - return True + return True def _create_full_composite_object( self, diff --git a/python/PiFinder/comet_catalog.py b/python/PiFinder/comet_catalog.py index f6f8c3b50..70af3b9fa 100644 --- a/python/PiFinder/comet_catalog.py +++ b/python/PiFinder/comet_catalog.py @@ -16,6 +16,7 @@ from PiFinder.composite_object import CompositeObject, MagnitudeObject, SizeObject import PiFinder.comets as comets from PiFinder.utils import Timer, comet_file +from PiFinder import timez from PiFinder.calc_utils import sf_utils logger = logging.getLogger("CometCatalog") @@ -55,19 +56,12 @@ def __init__(self, dt: datetime.datetime, shared_state: SharedStateObj): self._timer.do_timed_task = self.do_timed_task self._timer.time_delay_seconds = lambda: self.time_delay_seconds - # Check if we need to download - want_download, reason = comets.check_if_comet_download_needed(comet_file) - - if want_download: - logger.info(f"Download needed: {reason}") - # Start download in background and wait for completion - download_thread = threading.Thread(target=self._download_once, daemon=True) - download_thread.start() - download_thread.join() # Wait for download to complete - - # Now try to initialize comets immediately (if GPS available) + # Existing elements stay usable while freshness is checked and a new + # file downloads in the background. if self.shared_state.altaz_ready() and os.path.exists(comet_file): - self.do_timed_task() # Initialize immediately + self.do_timed_task() + + threading.Thread(target=self._refresh_if_needed, daemon=True).start() # Start timer after initialization self._timer.start_timer() @@ -90,14 +84,12 @@ def get_age(self) -> Optional[int]: self._cached_file_mtime = os.path.getmtime(comet_file) # Get file modification time from cache - local_date = datetime.datetime.fromtimestamp( - self._cached_file_mtime, tz=pytz.UTC - ) + local_date = timez.utc_from_timestamp(self._cached_file_mtime) # Calculate age using GPS time now = self.shared_state.datetime() if now.tzinfo is None: - now = now.replace(tzinfo=pytz.UTC) + now = pytz.UTC.localize(now) age_days = (now - local_date).total_seconds() / 86400 return round(age_days) @@ -108,7 +100,7 @@ def get_status(self) -> CatalogStatus: current_state = CatalogState.DOWNLOADING elif not self.shared_state.altaz_ready(): current_state = CatalogState.NO_GPS - elif not self.initialized: + elif self.calculation_progress is not None or not self.initialized: current_state = CatalogState.CALCULATING else: current_state = CatalogState.READY @@ -142,7 +134,7 @@ def _download_once(self): try: - def progress_callback(progress: int): + def progress_callback(progress: Optional[int]): self.download_progress = progress self._is_downloading = True @@ -151,20 +143,27 @@ def progress_callback(progress: int): success, _, file_mtime = comets.comet_data_download( comet_file, progress_callback=progress_callback ) - self._is_downloading = False - self.download_progress = None - # Update cached mtime after download - use the timestamp from download if success and file_mtime is not None: self._cached_file_mtime = file_mtime + if self.shared_state.altaz_ready(): + self.do_timed_task() age = self.get_age() age_str = f"{age} days" if age is not None else "? days" logger.info(f"Download completed: success={success}, age={age_str}") return success finally: + self._is_downloading = False + self.download_progress = None self._download_lock.release() + def _refresh_if_needed(self): + want_download, reason = comets.check_if_comet_download_needed(comet_file) + if want_download: + logger.info("Comet download needed: %s", reason) + self._download_once() + def refresh(self): """ Trigger a refresh by checking if download is needed. @@ -172,11 +171,6 @@ def refresh(self): """ logger.info("Refresh called - checking if download needed") - # Clear existing objects immediately - if self.get_objects(): - self.clear_objects() - self.initialized = False - # Do the check and download in background thread to return immediately def refresh_task(): # Check if we need to download @@ -184,16 +178,10 @@ def refresh_task(): if want_download: logger.info(f"Refresh will download: {reason}") - # Delete file to trigger download - if os.path.exists(comet_file): - os.remove(comet_file) - logger.info("Deleted comet file") - - # Download self._download_once() else: logger.info(f"Refresh using existing file: {reason}") - # File is fresh, just reinitialize from existing file + # File is fresh, recalculate from the existing elements. if self.shared_state.altaz_ready() and os.path.exists(comet_file): self.do_timed_task() @@ -231,9 +219,6 @@ def time_delay_seconds(self) -> int: def init_comets(self, dt): """Initialize comet catalog - called when GPS lock is available. Idempotent.""" logger.info("Starting comet calculation") - # Clear any existing objects to make this idempotent - if self.get_objects(): - self.clear_objects() def progress_callback(progress: int): self.calculation_progress = progress @@ -243,14 +228,20 @@ def progress_callback(progress: int): comet_dict = comets.calc_comets(dt, progress_callback=progress_callback) if not comet_dict: - self.initialized = False + # A failed refresh must not discard an already usable catalog. + self.initialized = bool(self.get_objects()) self.calculation_progress = None return - for sequence, (name, comet) in enumerate(comet_dict.items()): - self.add_comet(sequence, name, comet) + objects = [ + self._make_comet(sequence, name, comet) + for sequence, (name, comet) in enumerate(comet_dict.items()) + ] + self.replace_objects(objects) self._virtual_id_manager.mint_ids(self) + if self.catalog_filter is not None: + self.catalog_filter.mark_catalog_content_dirty() self.initialized = True self.calculation_progress = None # Clear progress after completion @@ -258,31 +249,36 @@ def progress_callback(progress: int): def add_comet(self, sequence: int, name: str, comet: Dict[str, Dict[str, float]]): """Add a single comet to the catalog""" try: - ra, dec = comet["radec"] - constellation = sf_utils.radec_to_constellation(ra, dec) - desc = f"Distance to\nEarth: {comet['earth_distance']:.2f} AU\nSun: {comet['sun_distance']:.2f} AU" - - mag = MagnitudeObject([comet.get("mag", [])]) - obj = CompositeObject.from_dict( - { - "id": -1, - "obj_type": "CM", - "ra": ra, - "dec": dec, - "const": constellation, - "size": SizeObject([]), - "mag": mag, - "mag_str": mag.calc_two_mag_representation(), - "names": [name], - "catalog_code": "CM", - "sequence": sequence + 1, - "description": desc, - } - ) - self.add_object(obj) + self.add_object(self._make_comet(sequence, name, comet)) except (KeyError, ValueError) as e: logger.error(f"Error adding comet {name}: {e}") + def _make_comet( + self, sequence: int, name: str, comet: Dict[str, Dict[str, float]] + ) -> CompositeObject: + ra, dec = comet["radec"] + constellation = sf_utils.radec_to_constellation(ra, dec) + desc = f"Distance to\nEarth: {comet['earth_distance']:.2f} AU\nSun: {comet['sun_distance']:.2f} AU" + mag = MagnitudeObject([comet.get("mag", [])]) + return CompositeObject.from_dict( + { + "id": -1, + "obj_type": "CM", + "ra": ra, + "dec": dec, + "const": constellation, + "size": SizeObject([]), + "mag": mag, + "mag_str": mag.calc_two_mag_representation(), + "names": [name], + "catalog_code": "CM", + "sequence": sequence + 1, + "description": desc, + "earth_distance_au": comet["earth_distance"], + "sun_distance_au": comet["sun_distance"], + } + ) + def do_timed_task(self): """Recalculate comet catalog periodically. diff --git a/python/PiFinder/comets.py b/python/PiFinder/comets.py index 2f9819608..ea309ff85 100644 --- a/python/PiFinder/comets.py +++ b/python/PiFinder/comets.py @@ -1,18 +1,17 @@ from typing import Dict, Any, Tuple, Optional, Callable -from datetime import datetime, timezone +from pathlib import Path from skyfield.data import mpc from skyfield.constants import GM_SUN_Pitjeva_2005_km3_s2 as GM_SUN from PiFinder.utils import Timer, comet_file from PiFinder.calc_utils import sf_utils -from PiFinder import timez +from PiFinder.download_utils import check_download_needed, download_atomic import numpy as np import pandas as pd -import requests -import os import logging import math logger = logging.getLogger("Comets") +COMET_VISIBLE_MAG_LIMIT = 15.0 def process_comet(comet_data, dt) -> Dict[str, Any]: @@ -35,7 +34,7 @@ def process_comet(comet_data, dt) -> Dict[str, Any]: + 2.5 * mag_k * math.log10(sun_distance.au) + 5.0 * math.log10(earth_distance.au) ) - if mag > 15: + if mag > COMET_VISIBLE_MAG_LIMIT: logger.debug(f"Filtering out {name}: mag={mag:.1f} (too dim)") return {} @@ -58,50 +57,20 @@ def process_comet(comet_data, dt) -> Dict[str, Any]: def check_if_comet_download_needed( local_filename, url=mpc.COMET_URL, timeout=5 ) -> Tuple[bool, str]: - """ - Check if comet data download is needed by comparing local file with remote. - - Args: - local_filename: Path to local file - url: URL to check - timeout: Request timeout in seconds - - Returns: - Tuple of (need_download: bool, reason: str) - """ - if not os.path.exists(local_filename): - return (True, "no existing file") - - try: - # Send a HEAD request to get headers without downloading - response = requests.head(url, timeout=timeout) - response.raise_for_status() - - last_modified = response.headers.get("Last-Modified") - if not last_modified: - return (False, "cannot verify remote date") + return check_download_needed(local_filename, url, timeout) - remote_date = datetime.strptime( - last_modified, "%a, %d %b %Y %H:%M:%S GMT" - ).replace(tzinfo=timezone.utc) - local_date = timez.utc_from_timestamp(os.path.getmtime(local_filename)) - - if remote_date > local_date: - age_diff = (remote_date - local_date).total_seconds() / 86400 - return (True, f"file outdated by {age_diff:.1f} days") - else: - return (False, "file is up to date") - - except requests.RequestException as e: - logger.warning(f"Could not check remote file: {e}") - return (False, f"network error: {e}") +def _validate_comet_file(path: Path) -> None: + with path.open("rb") as comet_data: + dataframe = mpc.load_comets_dataframe(comet_data) + if dataframe.empty: + raise ValueError("MPC comet file contains no objects") def comet_data_download( local_filename, url=mpc.COMET_URL, - progress_callback: Optional[Callable[[int], None]] = None, + progress_callback: Optional[Callable[[Optional[int]], None]] = None, ) -> Tuple[bool, Optional[float], Optional[float]]: """ Download comet data from the Minor Planet Center. @@ -109,55 +78,20 @@ def comet_data_download( Args: local_filename: Path to save the downloaded file url: URL to download from - progress_callback: Optional callback function that receives progress percentage (0-100) + progress_callback: Optional callback receiving a percentage (0-100), + or ``None`` when the server does not report a total size. Returns: Tuple of (success: bool, age_in_days: Optional[float], file_mtime: Optional[float]) file_mtime is the file's modification time as a timestamp (for caching) """ - try: - now = datetime.now(timezone.utc) - - logger.debug("Downloading comet data...") - response = requests.get(url, stream=True) - response.raise_for_status() - - # Get file size for progress calculation - total_size = int(response.headers.get("content-length", 0)) - downloaded = 0 - - with open(local_filename, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - f.write(chunk) - downloaded += len(chunk) - - # Report progress if callback provided and total size known - if progress_callback and total_size > 0: - progress = int((downloaded / total_size) * 100) - progress_callback(progress) - - # Try to get Last-Modified to set file mtime - last_modified = response.headers.get("Last-Modified") - if last_modified: - remote_date = datetime.strptime( - last_modified, "%a, %d %b %Y %H:%M:%S GMT" - ).replace(tzinfo=timezone.utc) - file_mtime = remote_date.timestamp() - os.utime(local_filename, (file_mtime, file_mtime)) - age_days = (now - remote_date).total_seconds() / 86400 - else: - file_mtime = os.path.getmtime(local_filename) - age_days = None - - logger.debug("File downloaded successfully.") - if progress_callback: - progress_callback(100) - return True, age_days, file_mtime - - except requests.RequestException as e: - logger.error(f"Error downloading comet data: {e}") - return False, None, None + result = download_atomic( + url, + local_filename, + progress_callback=progress_callback, + validator=_validate_comet_file, + ) + return result.success, result.age_days, result.file_mtime def _load_comets_dataframe() -> pd.DataFrame: @@ -287,11 +221,11 @@ def _calc_comets_vectorized(comets_df: pd.DataFrame, dt) -> Dict[str, Any]: names = comets_df["designation"].to_numpy() - # Keep comets that are NOT dimmer than mag 15. Phrased as ~(mag > 15) + # Keep comets that are NOT dimmer than the catalog safety limit. Phrased # rather than (mag <= 15) so NaN magnitudes are kept, matching the old # per-comet filter. comet_dict: Dict[str, Any] = {} - for i in np.nonzero(~(mag > 15))[0]: + for i in np.nonzero(~(mag > COMET_VISIBLE_MAG_LIMIT))[0]: name = str(names[i]) comet_dict[name] = { "name": name, diff --git a/python/PiFinder/composite_object.py b/python/PiFinder/composite_object.py index 127d007ed..2e7fe69eb 100644 --- a/python/PiFinder/composite_object.py +++ b/python/PiFinder/composite_object.py @@ -3,7 +3,8 @@ import numpy as np import json import math -from typing import List, Union, cast +from datetime import date +from typing import List, Optional, Union, cast from PiFinder.utils import is_number @@ -262,6 +263,15 @@ class CompositeObject: _details_loaded: bool = field(default=False) image_name: str = field(default="") surface_brightness: float = field(default=0.0) + # Runtime solar-system metadata. Kept structured so lists can sort it; + # descriptions are presentation, never a data source. + earth_distance_au: Optional[float] = field(default=None) + sun_distance_au: Optional[float] = field(default=None) + angular_motion_arcsec_per_hour: Optional[float] = field(default=None) + opposition_date: Optional[date] = field(default=None) + opposition_kind: str = field(default="") + peak_magnitude: Optional[float] = field(default=None) + peak_date: Optional[date] = field(default=None) logged: bool = field(default=False) last_filtered_time: float = 0 last_filtered_result: bool = True diff --git a/python/PiFinder/config.py b/python/PiFinder/config.py index d54cf0f87..d8fa37365 100644 --- a/python/PiFinder/config.py +++ b/python/PiFinder/config.py @@ -32,7 +32,8 @@ def load_config(self): self.config_file_path = Path(utils.data_dir, "config.json") self.default_file_path = Path(utils.pifinder_dir, "default_config.json") - if not os.path.exists(self.config_file_path): + had_saved_config = os.path.exists(self.config_file_path) + if not had_saved_config: self._config_dict = {} else: with open(self.config_file_path, "r") as config_file: @@ -43,6 +44,8 @@ def load_config(self): with open(self.default_file_path, "r") as config_file: self._default_config_dict = json.load(config_file) + self._migrate_asteroid_filters(had_saved_config) + # Load the equipment config eq_config = self.get_option("equipment") if eq_config is None: @@ -84,6 +87,27 @@ def load_config(self): else: self.locations = locations.Locations.from_dict(loc_config) + def _migrate_asteroid_filters(self, had_saved_config: bool) -> None: + """Enable the new asteroid catalog once in persisted filter lists. + + Defaults already cover fresh installs. The marker prevents a later + user choice to disable asteroids from being undone on every startup. + """ + marker = "migration.asteroid_filter_v1" + if not had_saved_config or self._config_dict.get(marker): + return + + additions = ( + ("filter.selected_catalogs", "MP"), + ("filter.object_types", "AS"), + ) + for option, value in additions: + saved_values = self._config_dict.get(option) + if isinstance(saved_values, list) and value not in saved_values: + saved_values.append(value) + self._config_dict[marker] = True + self.dump_config() + def save_equipment(self): """ Saves the equipment object state diff --git a/python/PiFinder/download_utils.py b/python/PiFinder/download_utils.py new file mode 100644 index 000000000..6e7acbdd2 --- /dev/null +++ b/python/PiFinder/download_utils.py @@ -0,0 +1,141 @@ +"""Safe downloads for runtime catalog data files. + +Catalog updates are deliberately transactional: callers keep using the existing +file while bytes arrive in a sibling temporary file. Only a complete, +validated response replaces the active file. +""" + +from __future__ import annotations + +import logging +import os +import tempfile +import time +from datetime import timezone +from email.utils import parsedate_to_datetime +from pathlib import Path +from typing import Callable, NamedTuple, Optional + +import requests + + +logger = logging.getLogger("CatalogDownload") + +ProgressCallback = Callable[[Optional[int]], None] +Validator = Callable[[Path], None] +REQUEST_TIMEOUT = (5, 30) + + +class DownloadResult(NamedTuple): + success: bool + age_days: Optional[float] + file_mtime: Optional[float] + error: Optional[str] = None + + +def _remote_timestamp(headers) -> Optional[float]: + value = headers.get("Last-Modified") + if not value: + return None + try: + parsed = parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.timestamp() + + +def check_download_needed( + local_filename: Path | str, + url: str, + timeout: float = 5, +) -> tuple[bool, str]: + """Compare a local catalog file with the server's Last-Modified value.""" + local_path = Path(local_filename) + if not local_path.exists(): + return True, "no existing file" + + try: + response = requests.head(url, timeout=timeout) + response.raise_for_status() + except requests.RequestException as exc: + logger.warning("Could not check %s: %s", url, exc) + return False, f"network error: {exc}" + + remote_mtime = _remote_timestamp(response.headers) + if remote_mtime is None: + return False, "cannot verify remote date" + local_mtime = local_path.stat().st_mtime + if remote_mtime > local_mtime: + age_diff = (remote_mtime - local_mtime) / 86400.0 + return True, f"file outdated by {age_diff:.1f} days" + return False, "file is up to date" + + +def download_atomic( + url: str, + local_filename: Path | str, + progress_callback: Optional[ProgressCallback] = None, + validator: Optional[Validator] = None, + timeout=REQUEST_TIMEOUT, +) -> DownloadResult: + """Download and validate ``url`` before atomically replacing the local file. + + ``progress_callback`` receives an integer percentage when Content-Length is + known, otherwise ``None`` to request an indeterminate progress indicator. + """ + local_path = Path(local_filename) + local_path.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Optional[Path] = None + try: + response = requests.get(url, stream=True, timeout=timeout) + response.raise_for_status() + total_size = int(response.headers.get("content-length", 0) or 0) + downloaded = 0 + if progress_callback: + progress_callback(0 if total_size else None) + + with tempfile.NamedTemporaryFile( + mode="wb", + prefix=f".{local_path.name}.", + suffix=".tmp", + dir=local_path.parent, + delete=False, + ) as temporary: + temporary_path = Path(temporary.name) + for chunk in response.iter_content(chunk_size=8192): + if not chunk: + continue + temporary.write(chunk) + downloaded += len(chunk) + if progress_callback and total_size: + progress_callback(min(99, int(downloaded * 100 / total_size))) + temporary.flush() + os.fsync(temporary.fileno()) + + if downloaded == 0: + raise ValueError("downloaded file is empty") + if validator: + validator(temporary_path) + + remote_mtime = _remote_timestamp(response.headers) + if remote_mtime is not None: + os.utime(temporary_path, (remote_mtime, remote_mtime)) + os.replace(temporary_path, local_path) + temporary_path = None + + file_mtime = local_path.stat().st_mtime + age_days = (time.time() - file_mtime) / 86400.0 + if progress_callback: + progress_callback(100) + return DownloadResult(True, age_days, file_mtime) + except (OSError, ValueError, requests.RequestException) as exc: + logger.error("Could not download %s: %s", url, exc) + return DownloadResult(False, None, None, str(exc)) + finally: + if temporary_path is not None: + try: + temporary_path.unlink() + except FileNotFoundError: + pass diff --git a/python/PiFinder/obj_types.py b/python/PiFinder/obj_types.py index a2a7cfa13..16efb3a17 100644 --- a/python/PiFinder/obj_types.py +++ b/python/PiFinder/obj_types.py @@ -21,6 +21,7 @@ def _(key: str) -> str: "Ast": _("Asterism"), # TRANSLATORS: Object type "Pla": _("Planet"), # TRANSLATORS: Object type "CM": _("Comet"), # TRANSLATORS: Object type + "AS": _("Asteroid"), # TRANSLATORS: Object type "?": _("Unkn"), # TRANSLATORS: Object type } @@ -34,6 +35,7 @@ def _(key: str) -> str: "***": "dstar", "Ast": "ast", "Pla": "planet", + "AS": "asteroid", } # abbreviations and symbols as used in the NGC/IC catalogues diff --git a/python/PiFinder/obslist_formats.py b/python/PiFinder/obslist_formats.py index 1ac8f1175..cf0522f31 100644 --- a/python/PiFinder/obslist_formats.py +++ b/python/PiFinder/obslist_formats.py @@ -211,6 +211,7 @@ def _parse_catalog_name(name: str) -> tuple[str, int]: "Ast": "ASTERISM", "Pla": "STAR", "CM": "COMET", + "AS": "ASTEROID", "?": "USER", } ARGO_TYPE_MAP_INV: dict[str, str] = {} @@ -233,6 +234,7 @@ def _parse_catalog_name(name: str) -> tuple[str, int]: "Ast": "Asterism", "Pla": "Star", "CM": "Star", + "AS": "Star", "?": "Star", } CELESTRON_TYPE_MAP_INV: dict[str, str] = {} @@ -252,7 +254,7 @@ def _parse_catalog_name(name: str) -> tuple[str, int]: def _skylist_object_id(obj_type: str) -> str: if obj_type in ("*", "D*", "***"): return "2,-1,-1" - if obj_type == "Pla": + if obj_type in ("Pla", "AS"): return "1,-1,-1" return "4,-1,-1" @@ -534,7 +536,8 @@ def read_text(text: str) -> ObsList: "asterism": "Ast", "planet": "Pla", "moon": "Pla", - "minor planet": "Pla", + "minor planet": "AS", + "asteroid": "AS", "dwarf planet": "Pla", "comet": "CM", "region of the sky": "?", diff --git a/python/PiFinder/ui/marking_menus.py b/python/PiFinder/ui/marking_menus.py index 19391cffc..93d9ad542 100644 --- a/python/PiFinder/ui/marking_menus.py +++ b/python/PiFinder/ui/marking_menus.py @@ -22,6 +22,7 @@ class MarkingMenuOption: selected: bool = False # shade bg? callback: Any = None menu_jump: Union[None, str] = None + value: Any = None def __str__(self): return self.label diff --git a/python/PiFinder/ui/menu_structure.py b/python/PiFinder/ui/menu_structure.py index a220feee7..c7ca88151 100644 --- a/python/PiFinder/ui/menu_structure.py +++ b/python/PiFinder/ui/menu_structure.py @@ -115,6 +115,12 @@ def _(key: str) -> Any: "objects": "catalog", "value": "CM", }, + { + "name": _("Asteroids"), + "class": UIObjectList, + "objects": "catalog", + "value": "MP", + }, { "name": _("NGC"), "class": UIObjectList, @@ -320,6 +326,10 @@ def _(key: str) -> Any: "name": _("Comets"), "value": "CM", }, + { + "name": _("Asteroids"), + "value": "MP", + }, { "name": _("NGC"), "value": "NGC", diff --git a/python/PiFinder/ui/object_list.py b/python/PiFinder/ui/object_list.py index 79dd2381c..d5f7b223f 100644 --- a/python/PiFinder/ui/object_list.py +++ b/python/PiFinder/ui/object_list.py @@ -13,6 +13,8 @@ import functools from functools import cache import math as math +import datetime +import time from PIL import Image, ImageChops from itertools import cycle @@ -58,7 +60,34 @@ class SortOrder(Enum): CATALOG_SEQUENCE = 0 # By catalog/sequence NEAREST = 1 # By Distance to target + BRIGHTEST = 2 # By apparent magnitude RA = 3 # By RA + EARTH_DISTANCE = 4 # By physical distance from Earth + OPPOSITION = 5 # By next opposition / greatest elongation + + +def _sort_objects( + objects: list[CompositeObject], order: SortOrder +) -> list[CompositeObject]: + if order == SortOrder.CATALOG_SEQUENCE: + return list(objects) + if order == SortOrder.RA: + return sorted(objects, key=lambda obj: obj.ra) + if order == SortOrder.BRIGHTEST: + return sorted(objects, key=lambda obj: obj.mag.filter_mag) + if order == SortOrder.EARTH_DISTANCE: + return sorted( + objects, + key=lambda obj: obj.earth_distance_au + if obj.earth_distance_au is not None + else math.inf, + ) + if order == SortOrder.OPPOSITION: + return sorted( + objects, + key=lambda obj: obj.opposition_date or datetime.date.max, + ) + return list(objects) class UIObjectList(UITextMenu): @@ -89,6 +118,7 @@ def __init__(self, *args, **kwargs) -> None: self._menu_items: list[CompositeObject] = [] self.catalog_info_1: str = "" self.catalog_info_2: str = "" + self.catalog_data_label: str = "" self._was_loading: bool = False # Track loading state to detect completion # Init display mode defaults @@ -125,27 +155,41 @@ def __init__(self, *args, **kwargs) -> None: # Base marking menu marking_menu_down = MarkingMenuOption() - # Add refresh option for comet catalog only - if ( - self.item_definition.get("objects") == "catalog" - and self.item_definition.get("value") == "CM" - ): + # Downloaded dynamic catalogs can refresh without discarding their + # currently displayed objects. + if self.item_definition.get( + "objects" + ) == "catalog" and self.item_definition.get("value") in ("CM", "MP"): marking_menu_down = MarkingMenuOption( label=_("Refresh"), - callback=self.mm_refresh_comets, # TRANSLATORS: Marking menu option to refresh comet catalog + callback=self.mm_refresh_dynamic_catalog, ) + asteroid_list = self.item_definition.get("value") == "MP" + self.marking_menu = MarkingMenu( left=MarkingMenuOption( label=_("Sort"), callback=MarkingMenu( - up=MarkingMenuOption(), + up=MarkingMenuOption( + label=_("MAG"), + callback=self.mm_change_sort, + value=SortOrder.BRIGHTEST, + ), left=MarkingMenuOption( - label=_("Nearest"), callback=self.mm_change_sort + label=_("NEAR"), + callback=self.mm_change_sort, + value=SortOrder.NEAREST, + ), + down=MarkingMenuOption( + label=_("OPP") if asteroid_list else _("RA"), + callback=self.mm_change_sort, + value=(SortOrder.OPPOSITION if asteroid_list else SortOrder.RA), ), - down=MarkingMenuOption(), right=MarkingMenuOption( - label=_("Standard"), callback=self.mm_change_sort + label=_("STD"), + callback=self.mm_change_sort, + value=SortOrder.CATALOG_SEQUENCE, ), ), ), @@ -191,8 +235,12 @@ def refresh_object_list(self, force_update=False): for catalog in self.catalogs.get_catalogs(only_selected=False): if catalog.catalog_code == self.item_definition["value"]: self._menu_items = catalog.get_filtered_objects() - age = catalog.get_age() - self.catalog_info_2 = "" if age is None else str(round(age, 0)) + self.catalog_data_label = catalog.get_data_label() or "" + if self.catalog_data_label: + self.catalog_info_2 = "" + else: + age = catalog.get_age() + self.catalog_info_2 = "" if age is None else str(round(age, 0)) if self.item_definition["objects"] == "recent": self._menu_items = self.ui_state.recent_list() @@ -209,9 +257,25 @@ def refresh_object_list(self, force_update=False): self.catalog_info_1 = str(self.get_nr_of_menu_items()) self._menu_items_sorted = self._menu_items - self.sort() + self.sort(show_message=False) - def _get_catalog_status_message(self) -> Tuple[Optional[str], Optional[int]]: + def _get_catalog_status(self): + if self.item_definition.get("objects") != "catalog": + return None + catalog = self.catalogs.get_catalog_by_code(self.item_definition.get("value")) + if catalog is None: + return None + status = catalog.get_status() + if ( + status.previous != CatalogState.READY + and status.current == CatalogState.READY + ): + self.refresh_object_list(force_update=True) + return status + + def _get_catalog_status_message( + self, status=None + ) -> Tuple[Optional[str], Optional[int]]: """ Generate status message explaining why catalog might be empty. Returns tuple of (message, progress_percentage). @@ -219,72 +283,36 @@ def _get_catalog_status_message(self) -> Tuple[Optional[str], Optional[int]]: Also handles refreshing object list when catalog transitions to READY. """ - if self.item_definition.get("objects") != "catalog": + status = status or self._get_catalog_status() + if status is None: return (None, None) - - catalog_code = self.item_definition.get("value") - if not catalog_code: + progress = status.data.get("progress") if status.data else None + if status.current == CatalogState.READY: return (None, None) - - for catalog in self.catalogs.get_catalogs(only_selected=False): - if catalog.catalog_code == catalog_code: - status = catalog.get_status() - - # Handle state transitions - refresh immediately when transitioning to READY - if ( - status.previous != CatalogState.READY - and status.current == CatalogState.READY - ): - self.refresh_object_list(force_update=True) - - # Extract progress if available - progress = None - if status.data and "progress" in status.data: - progress = status.data["progress"] - - # Map state to user-facing messages - if status.current == CatalogState.READY: - return (None, None) - elif status.current == CatalogState.DOWNLOADING: - return ( - _( - "Downloading..." - ), # TRANSLATORS: Status when catalog data is downloading - progress, - ) - elif status.current == CatalogState.NO_GPS: - return ( - _( - "No GPS lock" - ), # TRANSLATORS: Status when waiting for GPS position - None, - ) - elif status.current == CatalogState.CALCULATING: - return ( - _( - "Calculating..." - ), # TRANSLATORS: Status when computing object positions - progress, - ) - elif status.current == CatalogState.ERROR: - return (_("Error"), None) # TRANSLATORS: Generic error status - else: - return ( - _("Loading..."), - None, - ) # TRANSLATORS: Generic loading status - - return (None, None) - - def sort(self) -> None: + if status.current == CatalogState.DOWNLOADING: + return (_("Downloading..."), progress) + if status.current == CatalogState.NO_GPS: + return (_("No GPS lock"), None) + if status.current == CatalogState.CALCULATING: + return (_("Calculating..."), progress) + if status.current == CatalogState.ERROR: + return (_("Error"), None) + return (_("Loading..."), None) + + def sort(self, show_message: bool = True) -> None: + sort_labels = { + SortOrder.CATALOG_SEQUENCE: _("Catalog"), + SortOrder.NEAREST: _("Nearby"), + SortOrder.BRIGHTEST: _("Brightest"), + SortOrder.RA: _("RA"), + SortOrder.EARTH_DISTANCE: _("Distance"), + SortOrder.OPPOSITION: _("Opposition"), + } message = _("Sorting by\n{sort_order}").format( - sort_order=_("RA") - if self.current_sort == SortOrder.RA - else _("Catalog") - if self.current_sort == SortOrder.CATALOG_SEQUENCE - else _("Nearby") + sort_order=sort_labels[self.current_sort] ) - self.message(message, 0.1) + if show_message: + self.message(message, 0.1) self.update() if self.current_sort == SortOrder.NEAREST: @@ -300,8 +328,8 @@ def sort(self) -> None: self.nearby_refresh() self._current_item_index = 0 - if self.current_sort == SortOrder.CATALOG_SEQUENCE: - self._menu_items_sorted = self._menu_items + if self.current_sort != SortOrder.NEAREST: + self._menu_items_sorted = _sort_objects(self._menu_items, self.current_sort) self._current_item_index = 0 self.update() @@ -495,6 +523,29 @@ def active(self): else: self.refresh_object_list() + def _draw_download_progress(self, progress: Optional[int], intensity: int) -> None: + """Draw a compact determinate/indeterminate bar beside catalog age.""" + width = max(18, min(36, self.display.width // 4)) + height = 4 + x = self.display.width - width - 2 + y = self.line_position(0) + self.fonts.bold.height - height + color = self.colors.get(intensity) + self.draw.rectangle((x, y, x + width, y + height), outline=color) + inner_width = width - 2 + if progress is None: + segment = max(3, inner_width // 4) + offset = int(time.monotonic() * 8) % max(1, inner_width - segment + 1) + self.draw.rectangle( + (x + 1 + offset, y + 1, x + offset + segment, y + height - 1), + fill=color, + ) + else: + filled = round(inner_width * max(0, min(100, progress)) / 100) + if filled: + self.draw.rectangle( + (x + 1, y + 1, x + filled, y + height - 1), fill=color + ) + def update(self, force: bool = False) -> None: self.clear_screen() @@ -515,10 +566,17 @@ def update(self, force: bool = False) -> None: else: self._was_loading = is_loading + catalog_filter = self.catalogs.catalog_filter + if catalog_filter is not None and catalog_filter.is_dirty(): + self.refresh_object_list() + + # Poll dynamic-catalog state even while objects remain populated: an + # update keeps serving the old catalog and reports download progress. + catalog_status = self._get_catalog_status() # no objects to display if self.get_nr_of_menu_items() == 0: # Get catalog-specific status message if available - status_msg, progress = self._get_catalog_status_message() + status_msg, progress = self._get_catalog_status_message(catalog_status) # Re-check menu items in case refresh happened during status check if self.get_nr_of_menu_items() > 0: @@ -566,29 +624,44 @@ def update(self, force: bool = False) -> None: # Draw sorting mode in the empty rows above the focus line if self._current_item_index < half: intensity: int = int(64 + (((half - 1) - self._current_item_index) * 32.0)) - self.draw.text( - (begin_x, self.line_position(0)), - _("{catalog_info_1} obj").format( - catalog_info_1=self.catalog_info_1 - ) # TRANSLATORS: number of objects in object list - + _(", {catalog_info_2}d old").format( + catalog_header = _("{catalog_info_1} obj").format( + catalog_info_1=self.catalog_info_1 + ) + if self.catalog_data_label: + catalog_header += f", {self.catalog_data_label}" + elif self.catalog_info_2: + catalog_header += _(", {catalog_info_2}d old").format( catalog_info_2=self.catalog_info_2 ) - if self.catalog_info_2 - else "", # TRANSLATORS: suffix to number of objects in object list (indicating age of catalog data) + self.draw.text( + (begin_x, self.line_position(0)), + catalog_header, font=self.fonts.bold.font, fill=self.colors.get(intensity), ) self.draw.text( (begin_x, self.line_position(1)), _("Sort: {sort_order}").format( - sort_order=_("Catalog") - if self.current_sort == SortOrder.CATALOG_SEQUENCE - else _("Nearby") + sort_order={ + SortOrder.CATALOG_SEQUENCE: _("Catalog"), + SortOrder.NEAREST: _("Nearby"), + SortOrder.BRIGHTEST: _("Brightest"), + SortOrder.RA: _("RA"), + SortOrder.EARTH_DISTANCE: _("Distance"), + SortOrder.OPPOSITION: _("Opposition"), + }[self.current_sort] ), font=self.fonts.bold.font, fill=self.colors.get(intensity), ) + if ( + catalog_status is not None + and catalog_status.current == CatalogState.DOWNLOADING + ): + progress = ( + catalog_status.data.get("progress") if catalog_status.data else None + ) + self._draw_download_progress(progress, intensity) # Draw current selection hint self.draw.rectangle(layout.selection_box, outline=self.colors.get(128), width=1) line_number, line_pos = 0, 0 @@ -822,21 +895,15 @@ def mm_change_sort(self, marking_menu, menu_item): marking_menu.select_none() menu_item.selected = True - if menu_item.label == _("Nearest"): - self.current_sort = SortOrder.NEAREST - self.nearby_refresh() - self.sort() - return True + sort_order = getattr(menu_item, "value", None) + if not isinstance(sort_order, SortOrder): + return False - if menu_item.label == _("Standard"): - self.current_sort = SortOrder.CATALOG_SEQUENCE - self.sort() - return True - - if menu_item.label == _("RA"): - self.current_sort = SortOrder.RA - self.sort() - return True + self.current_sort = sort_order + if sort_order == SortOrder.NEAREST: + self.nearby_refresh() + self.sort() + return True def mm_jump_to_filter(self, marking_menu, menu_item): pass @@ -866,19 +933,20 @@ def serialize_ui_state(self) -> dict: else str(self.current_sort), "catalog_info_1": self.catalog_info_1, "catalog_info_2": self.catalog_info_2, + "catalog_data_label": self.catalog_data_label, } except Exception as e: return {"error": f"Failed to serialize object list state: {str(e)}"} - def mm_refresh_comets(self, marking_menu, menu_item): - """Force refresh of comet data from the internet""" - catalog = self.catalogs.get_catalog_by_code("CM") + def mm_refresh_dynamic_catalog(self, marking_menu, menu_item): + """Refresh downloaded elements while retaining the active objects.""" + catalog = self.catalogs.get_catalog_by_code(self.item_definition.get("value")) if catalog and hasattr(catalog, "refresh"): self.message( _("Refreshing..."), 1 ) # TRANSLATORS: Status message when refreshing comet catalog catalog.refresh() - # Clear the UI object list and refresh to show status + # Keep the current objects visible and refresh the status header. self.refresh_object_list(force_update=True) return True diff --git a/python/PiFinder/utils.py b/python/PiFinder/utils.py index 15cccfbb0..8ba6b5c51 100644 --- a/python/PiFinder/utils.py +++ b/python/PiFinder/utils.py @@ -132,6 +132,7 @@ def get_version() -> str: debug_dump_dir = data_dir / "solver_debug_dumps" comet_file = data_dir / "comets.txt" +asteroid_data_dir = data_dir / "asteroids" # Logging-config presets ship read-only in the source tree; the user's active # selection is persisted in the writable data dir (like config.json), stored as diff --git a/python/tests/data/asteroids_fixture.txt b/python/tests/data/asteroids_fixture.txt new file mode 100644 index 000000000..df7508e5d --- /dev/null +++ b/python/tests/data/asteroids_fixture.txt @@ -0,0 +1,10 @@ +00001 3.35 0.15 K25BL 231.53975 73.29974 80.24963 10.58789 0.0795763 0.21429712 2.7656157 0 MPO950947 7369 126 1801-2025 0.69 M-v 30k MPC 0000 (1) Ceres +00002 4.11 0.15 K25BL 211.52977 310.93340 172.88859 34.92833 0.2306430 0.21379713 2.7699258 0 MPO950947 8934 124 1804-2025 0.64 M-c 28k MPC 0000 (2) Pallas +00003 5.19 0.15 K25BL 217.59095 247.88367 169.81989 12.98604 0.2558258 0.22579938 2.6708791 0 MPO937415 7565 118 1804-2025 0.67 M-v 3Ek MPC 0000 (3) Juno +00004 3.25 0.15 K25BL 26.80968 151.53712 103.70232 7.14406 0.0901676 0.27158812 2.3615413 0 MPO925791 7543 112 1821-2025 0.69 M-p 18k MPC 0000 (4) Vesta +00005 6.97 0.15 K25BL 133.86760 359.34517 141.44862 5.35925 0.1875086 0.23826852 2.5768646 0 MPO950947 3339 89 1845-2025 0.79 M-v 3Ek MPC 0000 (5) Astraea +00006 5.61 0.15 K25BL 352.56367 239.69622 138.61473 14.73615 0.2022301 0.26092182 2.4254693 0 MPO950947 6086 105 1848-2025 0.65 M-v 3Ek MPC 0000 (6) Hebe +00007 5.67 0.15 K25BL 61.72502 145.48204 259.49459 5.51881 0.2302133 0.26733389 2.3865290 0 MPO950947 5322 91 1848-2025 0.69 M-v 3Ek MPC 0000 (7) Iris +00008 6.61 0.15 K25BL 198.90078 285.42673 110.84339 5.89033 0.1563337 0.30177489 2.2013072 0 MPO937415 2806 95 1847-2025 0.78 M-v 3Ek MPC 0000 (8) Flora +66146 14.36 0.15 K25BL 74.76748 84.99252 101.87869 5.41525 0.4836695 1.41022421 0.7875484 0 MPO888142 2190 26 1982-2024 0.65 M-v 3Ek MPC 0000 (66146) +F2637 17.87 0.15 K25BL 245.95188 16.59972 96.45769 16.72221 0.2085330 1.22534637 0.8648952 0 E2023-R01 363 11 1997-2022 0.81 M-v 3Ek MPC 0000 (152637) diff --git a/python/tests/test_asteroids.py b/python/tests/test_asteroids.py new file mode 100644 index 000000000..935855520 --- /dev/null +++ b/python/tests/test_asteroids.py @@ -0,0 +1,135 @@ +"""Asteroid source parsing, propagation, photometry, and apparition tests.""" + +import math +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np +import pytest +from skyfield.constants import GM_SUN_Pitjeva_2005_km3_s2 as GM_SUN +from skyfield.data import mpc + +import PiFinder.asteroids as asteroids +from PiFinder.calc_utils import sf_utils + + +FIXTURE = Path(__file__).parent / "data" / "asteroids_fixture.txt" +DT = datetime(2026, 7, 15, tzinfo=timezone.utc) + + +@pytest.fixture(autouse=True) +def observer(): + sf_utils.set_location(50.85, 4.35, 50.0) + + +def angular_separation_arcsec(ra1, dec1, ra2, dec2): + r1, d1, r2, d2 = map(math.radians, (ra1, dec1, ra2, dec2)) + a = ( + math.sin((d2 - d1) / 2) ** 2 + + math.cos(d1) * math.cos(d2) * math.sin((r2 - r1) / 2) ** 2 + ) + return math.degrees(2 * math.asin(min(1.0, math.sqrt(a)))) * 3600.0 + + +@pytest.mark.unit +def test_loads_standard_mpcorb_file_with_stable_numbers(): + dataframe = asteroids.load_asteroids_dataframe([FIXTURE]) + assert len(dataframe) == 10 + assert list(dataframe.number[:4]) == [1, 2, 3, 4] + assert dataframe.number.iloc[-1] == 152637 + assert asteroids.minor_planet_name(dataframe.designation.iloc[0]) == "Ceres" + assert asteroids.minor_planet_name(dataframe.designation.iloc[-1]) == "152637" + + +@pytest.mark.unit +def test_vectorized_positions_match_skyfield_per_object_oracle(): + dataframe = asteroids.load_asteroids_dataframe([FIXTURE]).iloc[:8] + calculated = asteroids._calculate_dataframe( + dataframe, DT, include_apparitions=False + ) + time = sf_utils.ts.from_datetime(DT) + sun = sf_utils.eph["sun"] + for _, row in dataframe.iterrows(): + asteroid = sun + mpc.mpcorb_orbit(row, sf_utils.ts, GM_SUN) + topocentric = (asteroid - sf_utils.observer_loc).at(time) + one_hour_later = sf_utils.ts.tt_jd(time.tt + 1.0 / 24.0) + topocentric_later = (asteroid - sf_utils.observer_loc).at(one_hour_later) + heliocentric = (asteroid - sun).at(time) + ra, dec, earth_distance = topocentric.radec(sf_utils.ts.J2000) + sun_distance = heliocentric.distance() + item = calculated[int(row.number)] + separation = angular_separation_arcsec( + ra.degrees, dec.degrees, item["radec"][0], item["radec"][1] + ) + assert separation < 0.05 + assert item["earth_distance"] == pytest.approx(earth_distance.au, rel=1e-7) + assert item["sun_distance"] == pytest.approx(sun_distance.au, rel=1e-7) + later_ra, later_dec, _ = topocentric_later.radec(sf_utils.ts.J2000) + oracle_motion = angular_separation_arcsec( + ra.degrees, dec.degrees, later_ra.degrees, later_dec.degrees + ) + assert item["angular_motion_arcsec_per_hour"] == pytest.approx( + oracle_motion, abs=0.01 + ) + + +@pytest.mark.unit +def test_hg_magnitude_is_h_at_unit_distances_and_zero_phase(): + magnitude = asteroids.hg_magnitude( + np.array([8.5]), + np.array([0.15]), + np.array([1.0]), + np.array([1.0]), + np.array([0.0]), + ) + assert magnitude[0] == pytest.approx(8.5) + + +@pytest.mark.unit +def test_apparition_reports_vesta_opposition_and_nearby_peak(): + dataframe = asteroids.load_asteroids_dataframe([FIXTURE]) + vesta = dataframe[dataframe.number == 4] + result = asteroids._calculate_dataframe(vesta, DT)[4] + assert result["opposition_kind"] == "Opposition" + assert result["opposition_date"].isoformat() == "2026-10-13" + assert abs((result["peak_date"] - result["opposition_date"]).days) <= 90 + assert result["peak_magnitude"] <= result["mag"] + + +@pytest.mark.unit +def test_next_apparition_skips_just_passed_day_zero_opposition(): + index, is_opposition = asteroids._next_apparition_index( + np.array([180.0, 175.0, 160.0, 170.0, 179.0, 170.0]) + ) + assert index == 4 + assert is_opposition + + +@pytest.mark.unit +def test_next_apparition_skips_day_zero_greatest_elongation(): + index, is_opposition = asteroids._next_apparition_index( + np.array([120.0, 110.0, 80.0, 100.0, 125.0, 100.0]) + ) + assert index == 4 + assert not is_opposition + + +@pytest.mark.unit +def test_visibility_cut_rejects_non_finite_and_dim_objects(): + dataframe = asteroids.load_asteroids_dataframe([FIXTURE]).iloc[:2].copy() + dataframe.loc[dataframe.index[0], "magnitude_H"] = np.nan + dataframe.loc[dataframe.index[1], "magnitude_H"] = 99.0 + assert asteroids._calculate_dataframe(dataframe, DT) == {} + + +@pytest.mark.unit +def test_calc_asteroids_without_observer_is_empty(): + saved_location = sf_utils.observer_loc + saved_last = sf_utils._last_location + try: + sf_utils.observer_loc = None + sf_utils._last_location = None + assert asteroids.calc_asteroids(DT, [FIXTURE]) == {} + finally: + sf_utils.observer_loc = saved_location + sf_utils._last_location = saved_last diff --git a/python/tests/test_catalog_filter_cache.py b/python/tests/test_catalog_filter_cache.py index c66ef54bf..d7a0674c8 100644 --- a/python/tests/test_catalog_filter_cache.py +++ b/python/tests/test_catalog_filter_cache.py @@ -9,7 +9,7 @@ import pytest -from PiFinder.catalogs import Catalog, CatalogFilter +from PiFinder.catalogs import Catalog, CatalogFilter, Catalogs from PiFinder.composite_object import CompositeObject, MagnitudeObject @@ -26,13 +26,15 @@ def altaz_ready(self): return False -def _make_obj(seq: int, mag: float = 10.0, logged: bool = False): +def _make_obj( + seq: int, mag: float = 10.0, logged: bool = False, catalog_code: str = "TST" +): return CompositeObject( id=seq, object_id=seq, ra=10.0 * seq, dec=1.0 * seq, - catalog_code="TST", + catalog_code=catalog_code, sequence=seq, description=f"obj {seq}", mag=MagnitudeObject([mag]), @@ -116,3 +118,30 @@ def test_object_mutation_applied_after_mark_dirty(catalog): catalog.get_objects()[1].logged = True catalog.catalog_filter.mark_dirty() assert _sequences(catalog.filter_objects()) == [1, 3] + + +@pytest.mark.unit +def test_dynamic_content_change_refilters_only_changed_catalog(): + changed = Catalog("DYN", "dynamic") + unchanged = Catalog("STATIC", "static") + changed.add_object(_make_obj(1, mag=5.0, catalog_code="DYN")) + unchanged.add_object(_make_obj(1, mag=5.0, catalog_code="STATIC")) + catalog_filter = CatalogFilter(shared_state=FakeSharedState(), magnitude=10.0) + catalogs = Catalogs([changed, unchanged]) + catalogs.set_catalog_filter(catalog_filter) + catalogs.filter_catalogs() + changed_first = changed.get_filtered_objects() + unchanged_first = unchanged.get_filtered_objects() + dirty_time = catalog_filter.dirty_time + + changed.get_objects()[0].mag = MagnitudeObject([15.0]) + changed.invalidate_filter_cache() + catalog_filter.mark_catalog_content_dirty() + assert catalog_filter.is_dirty() + catalogs.filter_catalogs() + + assert catalog_filter.dirty_time == dirty_time + assert changed.get_filtered_objects() is not changed_first + assert changed.get_filtered_objects() == [] + assert unchanged.get_filtered_objects() is unchanged_first + assert not catalog_filter.is_dirty() diff --git a/python/tests/test_config_asteroid_migration.py b/python/tests/test_config_asteroid_migration.py new file mode 100644 index 000000000..c72a10a89 --- /dev/null +++ b/python/tests/test_config_asteroid_migration.py @@ -0,0 +1,47 @@ +"""One-time migration for persisted asteroid filter selections.""" + +import json + +import pytest + +from PiFinder import utils +from PiFinder.config import Config + + +def write_config_files(tmp_path, saved): + (tmp_path / "default_config.json").write_text( + json.dumps( + { + "filter.selected_catalogs": ["NGC", "MP"], + "filter.object_types": ["Gx", "AS"], + } + ) + ) + data_dir = tmp_path / "data" + data_dir.mkdir() + (data_dir / "config.json").write_text(json.dumps(saved)) + return data_dir + + +@pytest.mark.unit +def test_existing_saved_filters_gain_asteroids_once(tmp_path, monkeypatch): + data_dir = write_config_files( + tmp_path, + { + "filter.selected_catalogs": ["NGC"], + "filter.object_types": ["Gx"], + }, + ) + monkeypatch.setattr(utils, "data_dir", data_dir) + monkeypatch.setattr(utils, "pifinder_dir", tmp_path) + + config = Config() + assert config.get_option("filter.selected_catalogs") == ["NGC", "MP"] + assert config.get_option("filter.object_types") == ["Gx", "AS"] + + # Once migrated, an explicit user choice remains authoritative. + config.set_option("filter.selected_catalogs", ["NGC"]) + config.set_option("filter.object_types", ["Gx"]) + reloaded = Config() + assert reloaded.get_option("filter.selected_catalogs") == ["NGC"] + assert reloaded.get_option("filter.object_types") == ["Gx"] diff --git a/python/tests/test_download_utils.py b/python/tests/test_download_utils.py new file mode 100644 index 000000000..a0ee567b8 --- /dev/null +++ b/python/tests/test_download_utils.py @@ -0,0 +1,103 @@ +"""Transactional runtime catalog download tests.""" + +from pathlib import Path + +import pytest +import requests + +from PiFinder import download_utils + + +class FakeResponse: + def __init__(self, chunks, headers=None): + self.chunks = chunks + self.headers = headers or {} + + def raise_for_status(self): + return None + + def iter_content(self, chunk_size): + return iter(self.chunks) + + +@pytest.mark.unit +def test_atomic_download_replaces_only_after_validation(tmp_path, monkeypatch): + destination = tmp_path / "catalog.txt" + destination.write_bytes(b"old catalog") + monkeypatch.setattr( + download_utils.requests, + "get", + lambda *args, **kwargs: FakeResponse( + [b"new ", b"catalog"], {"content-length": "11"} + ), + ) + observed_during_validation = [] + + def validate(path: Path): + observed_during_validation.append(destination.read_bytes()) + assert path.read_bytes() == b"new catalog" + + progress = [] + result = download_utils.download_atomic( + "https://example.test/catalog", + destination, + progress_callback=progress.append, + validator=validate, + ) + assert result.success + assert observed_during_validation == [b"old catalog"] + assert destination.read_bytes() == b"new catalog" + assert progress[0] == 0 + assert progress[-1] == 100 + + +@pytest.mark.unit +def test_failed_validation_preserves_old_catalog(tmp_path, monkeypatch): + destination = tmp_path / "catalog.txt" + destination.write_bytes(b"known good") + monkeypatch.setattr( + download_utils.requests, + "get", + lambda *args, **kwargs: FakeResponse([b"broken"]), + ) + + def reject(_path): + raise ValueError("bad catalog") + + result = download_utils.download_atomic( + "https://example.test/catalog", destination, validator=reject + ) + assert not result.success + assert destination.read_bytes() == b"known good" + assert not list(tmp_path.glob("*.tmp")) + + +@pytest.mark.unit +def test_unknown_content_length_reports_indeterminate_progress(tmp_path, monkeypatch): + monkeypatch.setattr( + download_utils.requests, + "get", + lambda *args, **kwargs: FakeResponse([b"content"]), + ) + progress = [] + result = download_utils.download_atomic( + "https://example.test/catalog", + tmp_path / "catalog.txt", + progress_callback=progress.append, + ) + assert result.success + assert progress == [None, 100] + + +@pytest.mark.unit +def test_network_failure_preserves_old_catalog(tmp_path, monkeypatch): + destination = tmp_path / "catalog.txt" + destination.write_bytes(b"old") + + def fail(*args, **kwargs): + raise requests.Timeout("timed out") + + monkeypatch.setattr(download_utils.requests, "get", fail) + result = download_utils.download_atomic("https://example.test/catalog", destination) + assert not result.success + assert destination.read_bytes() == b"old" diff --git a/python/tests/test_dynamic_catalogs.py b/python/tests/test_dynamic_catalogs.py new file mode 100644 index 000000000..aae205b5e --- /dev/null +++ b/python/tests/test_dynamic_catalogs.py @@ -0,0 +1,143 @@ +"""Dynamic catalog identity, refresh retention, and progress-state tests.""" + +import datetime + +import pytest + +from PiFinder.asteroid_catalog import AsteroidCatalog +from PiFinder.catalog_base import CatalogState +from PiFinder.catalogs import Catalog +from PiFinder.comet_catalog import CometCatalog +from PiFinder.composite_object import CompositeObject + + +class ReadySharedState: + def altaz_ready(self): + return True + + def datetime(self): + return datetime.datetime(2026, 7, 15, tzinfo=datetime.timezone.utc) + + +def initialized_catalog(cls, code): + catalog = cls.__new__(cls) + Catalog.__init__(catalog, code, "test") + catalog.shared_state = ReadySharedState() + catalog._last_state = CatalogState.READY + catalog._is_downloading = False + catalog.download_progress = None + catalog.calculation_progress = None + catalog.initialized = True + return catalog + + +@pytest.mark.unit +def test_asteroid_object_uses_stable_number_and_structured_metadata(): + catalog = initialized_catalog(AsteroidCatalog, "MP") + asteroid = { + "number": 4, + "name": "Vesta", + "radec": (20.0, 5.0), + "mag": 6.5, + "earth_distance": 1.2, + "sun_distance": 2.2, + "angular_motion_arcsec_per_hour": 42.3, + "opposition_kind": "Opposition", + "opposition_date": datetime.date(2026, 10, 13), + "peak_magnitude": 6.4, + "peak_date": datetime.date(2026, 10, 12), + } + obj = catalog._make_object(asteroid) + assert obj.catalog_code == "MP" + assert obj.obj_type == "AS" + assert obj.sequence == 4 + assert obj.names == ["Vesta"] + assert obj.earth_distance_au == 1.2 + assert obj.opposition_date.isoformat() == "2026-10-13" + assert obj.description.splitlines()[:2] == [ + "Opp: 2026-10-13", + "Peak 6.4: 2026-10-12", + ] + assert obj.description.splitlines()[-1] == 'Motion: 42.3"/h' + + +@pytest.mark.unit +def test_asteroid_catalog_labels_annual_edition_instead_of_file_age(tmp_path): + catalog = initialized_catalog(AsteroidCatalog, "MP") + catalog.data_directory = tmp_path + (tmp_path / "Soft00Bright-2026.txt").touch() + assert catalog.get_data_label() == "MPC 2026" + + +@pytest.mark.unit +def test_asteroid_edition_label_uses_filename_before_gps_time(tmp_path): + catalog = initialized_catalog(AsteroidCatalog, "MP") + catalog.data_directory = tmp_path + catalog.shared_state.datetime = lambda: None + (tmp_path / "Soft00Bright-2026.txt").touch() + assert catalog.get_data_label() == "MPC 2026" + + +@pytest.mark.unit +def test_asteroid_edition_label_is_empty_without_gps_or_file(tmp_path): + catalog = initialized_catalog(AsteroidCatalog, "MP") + catalog.data_directory = tmp_path + catalog.shared_state.datetime = lambda: None + assert catalog.get_data_label() is None + + +@pytest.mark.unit +def test_asteroid_source_year_is_not_selected_before_gps(monkeypatch): + catalog = AsteroidCatalog.__new__(AsteroidCatalog) + catalog.shared_state = type("NoGpsState", (), {"altaz_ready": lambda self: False})() + monkeypatch.setattr( + "PiFinder.asteroid_catalog.asteroids.check_asteroid_download_needed", + lambda *_args, **_kwargs: pytest.fail("source year selected without GPS"), + ) + catalog._refresh_sources() + + +@pytest.mark.unit +def test_populated_asteroid_catalog_reports_download_progress(): + catalog = initialized_catalog(AsteroidCatalog, "MP") + catalog.add_object(CompositeObject(catalog_code="MP", sequence=4)) + catalog._is_downloading = True + catalog.download_progress = 42 + status = catalog.get_status() + assert catalog.get_count() == 1 + assert status.current == CatalogState.DOWNLOADING + assert status.data == {"progress": 42} + + +@pytest.mark.unit +def test_comet_refresh_keeps_old_objects_while_downloading(monkeypatch): + catalog = initialized_catalog(CometCatalog, "CM") + catalog.add_object(CompositeObject(catalog_code="CM", sequence=1)) + catalog._is_downloading = True + catalog.download_progress = 33 + status = catalog.get_status() + assert status.current == CatalogState.DOWNLOADING + assert status.data == {"progress": 33} + catalog._is_downloading = False + catalog.download_progress = None + monkeypatch.setattr( + "PiFinder.comet_catalog.comets.check_if_comet_download_needed", + lambda *_args, **_kwargs: (True, "new data"), + ) + + def download(): + assert catalog.get_count() == 1 + return False + + catalog._download_once = download + + class ImmediateThread: + def __init__(self, target, **kwargs): + self.target = target + + def start(self): + self.target() + + monkeypatch.setattr("PiFinder.comet_catalog.threading.Thread", ImmediateThread) + catalog.refresh() + assert catalog.get_count() == 1 diff --git a/python/tests/test_menu_struct.py b/python/tests/test_menu_struct.py index 39894427d..12fee0c33 100644 --- a/python/tests/test_menu_struct.py +++ b/python/tests/test_menu_struct.py @@ -47,6 +47,7 @@ def test_important_catalog_entries_exist(): assert "Planets" in catalog_names assert "Comets" in catalog_names + assert "Asteroids" in catalog_names assert "NGC" in catalog_names assert "Messier" in catalog_names diff --git a/python/tests/test_obj_types_docs.py b/python/tests/test_obj_types_docs.py index 73a97eda3..38e678fc6 100644 --- a/python/tests/test_obj_types_docs.py +++ b/python/tests/test_obj_types_docs.py @@ -14,8 +14,9 @@ from pathlib import Path import pytest +from PIL import Image -from PiFinder.obj_types import OBJ_TYPES +from PiFinder.obj_types import OBJ_TYPE_MARKERS, OBJ_TYPES _ROOT = Path(__file__).resolve().parents[2] _README = _ROOT / "docs/ax/catalog/obslist-formats/README.md" @@ -58,3 +59,13 @@ def test_default_config_object_types_match_obj_types(): assert set(config["filter.object_types"]) == set( OBJ_TYPES ), "default_config.json 'filter.object_types' is out of sync with OBJ_TYPES." + + +@pytest.mark.unit +def test_marker_mappings_have_11px_assets_and_asteroid_is_distinct(): + assert OBJ_TYPE_MARKERS["AS"] != OBJ_TYPE_MARKERS["Ast"] + for marker_name in set(OBJ_TYPE_MARKERS.values()): + marker_path = _ROOT / "markers" / f"mrk_{marker_name}.png" + assert marker_path.exists(), f"missing marker asset: {marker_path.name}" + with Image.open(marker_path) as marker: + assert marker.size == (11, 11) diff --git a/python/tests/test_object_list_progress.py b/python/tests/test_object_list_progress.py new file mode 100644 index 000000000..e93b37515 --- /dev/null +++ b/python/tests/test_object_list_progress.py @@ -0,0 +1,47 @@ +"""Compact progress-bar rendering used by populated dynamic catalogs.""" + +from types import SimpleNamespace + +import pytest +from PIL import Image, ImageDraw + +from PiFinder.ui.object_list import UIObjectList + + +class Colors: + def get(self, value): + return (value, 0, 0) + + +def progress_list(): + ui = UIObjectList.__new__(UIObjectList) + ui.display = SimpleNamespace(width=128) + ui.fonts = SimpleNamespace(bold=SimpleNamespace(height=9)) + ui.colors = Colors() + ui.line_position = lambda _line: 12 + image = Image.new("RGB", (128, 32)) + ui.draw = ImageDraw.Draw(image) + return ui, image + + +@pytest.mark.unit +def test_determinate_progress_bar_draws_outline_and_half_fill(): + ui, image = progress_list() + ui._draw_download_progress(50, 255) + # Right-aligned 32px bar on a 128px display; midpoint is filled but its + # far-right interior remains empty. + assert image.getpixel((100, 18))[0] == 255 + assert image.getpixel((122, 18))[0] == 0 + + +@pytest.mark.unit +def test_indeterminate_progress_bar_draws_activity(monkeypatch): + ui, image = progress_list() + monkeypatch.setattr("PiFinder.ui.object_list.time.monotonic", lambda: 0.0) + ui._draw_download_progress(None, 255) + red_pixels = sum( + image.getpixel((x, y))[0] > 0 + for x in range(image.width) + for y in range(image.height) + ) + assert red_pixels > 2 * 32 # outline plus a moving interior segment diff --git a/python/tests/test_object_list_sorting.py b/python/tests/test_object_list_sorting.py new file mode 100644 index 000000000..33672f39a --- /dev/null +++ b/python/tests/test_object_list_sorting.py @@ -0,0 +1,68 @@ +"""Object-list sorting for general and solar-system metadata.""" + +import datetime +from unittest.mock import Mock + +import pytest + +import PiFinder.i18n # noqa: F401 +from PiFinder.composite_object import CompositeObject, MagnitudeObject +from PiFinder.ui.object_list import SortOrder, UIObjectList, _sort_objects + + +def obj(sequence, mag, distance=None, opposition=None): + return CompositeObject( + catalog_code="MP", + sequence=sequence, + mag=MagnitudeObject([mag]), + earth_distance_au=distance, + opposition_date=opposition, + ) + + +@pytest.mark.unit +def test_brightest_sort_is_generic(): + objects = [obj(1, 12.0), obj(2, 7.0), obj(3, 10.0)] + assert [item.sequence for item in _sort_objects(objects, SortOrder.BRIGHTEST)] == [ + 2, + 3, + 1, + ] + + +@pytest.mark.unit +def test_earth_distance_sort_puts_unknown_last(): + objects = [obj(1, 1, 2.5), obj(2, 1, None), obj(3, 1, 0.4)] + assert [ + item.sequence for item in _sort_objects(objects, SortOrder.EARTH_DISTANCE) + ] == [3, 1, 2] + + +@pytest.mark.unit +def test_opposition_sort_puts_unknown_last(): + objects = [ + obj(1, 1, opposition=datetime.date(2027, 2, 1)), + obj(2, 1), + obj(3, 1, opposition=datetime.date(2026, 9, 1)), + ] + assert [item.sequence for item in _sort_objects(objects, SortOrder.OPPOSITION)] == [ + 3, + 1, + 2, + ] + + +@pytest.mark.unit +def test_automatic_list_resort_does_not_show_toast(): + ui = UIObjectList.__new__(UIObjectList) + ui.current_sort = SortOrder.CATALOG_SEQUENCE + ui._menu_items = [] + ui._menu_items_sorted = [] + ui._current_item_index = 0 + ui.message = Mock() + ui.update = Mock() + + ui.sort(show_message=False) + + ui.message.assert_not_called() + assert ui.update.called diff --git a/python/tests/test_obslist_formats.py b/python/tests/test_obslist_formats.py index 1e76a5df7..0460ff131 100644 --- a/python/tests/test_obslist_formats.py +++ b/python/tests/test_obslist_formats.py @@ -272,6 +272,8 @@ def _read_type(type_str): assert _read_type("Planetary Nebula") == "PN" assert _read_type("open star cluster") == "OC" + assert _read_type("asteroid") == "AS" + assert _read_type("minor planet") == "AS" # PiFinder codes (from our own v1.0 exports) pass through unchanged assert _read_type("Gx") == "Gx" # Unknown strings become '?' so the default Type filter still shows them diff --git a/python/tests/test_ui_modules.py b/python/tests/test_ui_modules.py index 84807ad9a..6370fdd85 100644 --- a/python/tests/test_ui_modules.py +++ b/python/tests/test_ui_modules.py @@ -361,8 +361,31 @@ def _no_comet_download(): """ import PiFinder.comets as comets - with mock.patch.object( - comets, "comet_data_download", return_value=(False, None, None) + with ( + mock.patch.object( + comets, + "check_if_comet_download_needed", + return_value=(False, "test environment"), + ), + mock.patch.object( + comets, "comet_data_download", return_value=(False, None, None) + ), + ): + yield + + +@pytest.fixture(scope="session", autouse=True) +def _no_asteroid_download(): + """Keep the UI harness hermetic while building AsteroidCatalog.""" + import PiFinder.asteroids as asteroids + + with ( + mock.patch.object( + asteroids, + "check_asteroid_download_needed", + return_value=(False, "test environment"), + ), + mock.patch.object(asteroids, "download_asteroid_year"), ): yield diff --git a/python/tests/website/test_web_remote.py b/python/tests/website/test_web_remote.py index 63548afa5..7d2117bee 100644 --- a/python/tests/website/test_web_remote.py +++ b/python/tests/website/test_web_remote.py @@ -406,7 +406,7 @@ def test_remote_nav_right(driver): press_keys_and_validate( driver, - "RDDD", + "RDDDD", expected_values={ "ui_type": "UITextMenu", "title": "By Catalog", @@ -542,7 +542,7 @@ def test_remote_backtotop(driver): press_keys_and_validate( driver, - "RDRDDDR31RW", # W = 1s wait to let async key callback and UI state update settle + "RDRDDDDR31RW", # W = 1s wait to let async key callback and UI state update settle expected_values={ "ui_type": "UIObjectDetails", "object": {"display_name": "M 31"}, @@ -567,7 +567,7 @@ def test_remote_markingmenu(driver): press_keys_and_validate( driver, - "RDRDDDR31RL", + "RDRDDDDR31RL", expected_values={ "current_item": "M 31", "display_mode": "LOCATE", @@ -644,7 +644,7 @@ def test_remote_recent(driver): # Navigate to M31 object details press_keys_and_validate( driver, - "RDRDDDR31RW", # W = 1s wait to let async key callback and UI state update settle + "RDRDDDDR31RW", # W = 1s wait to let async key callback and UI state update settle expected_values={ "ui_type": "UIObjectDetails", "object": {"display_name": "M 31"}, diff --git a/python/tests/website/test_web_remote_objects.py b/python/tests/website/test_web_remote_objects.py index 0006704ce..2f6f29b5a 100644 --- a/python/tests/website/test_web_remote_objects.py +++ b/python/tests/website/test_web_remote_objects.py @@ -17,7 +17,7 @@ This file adds coverage for the remaining Objects sub-items: - Objects > All Filtered - - Objects > By Catalog > Planets, Comets, NGC + - Objects > By Catalog > Planets, Comets, Asteroids, NGC - Objects > By Catalog > DSO... (nested submenu) - Objects > By Catalog > Stars... (nested submenu) - Objects > Custom (UIRADecEntry) @@ -34,10 +34,11 @@ By Catalog submenu (0-indexed): 0: Planets (UIObjectList, catalog "PL") 1: Comets (UIObjectList, catalog "CM") - 2: NGC (UIObjectList, catalog "NGC") - 3: Messier (UIObjectList, catalog "M") ← already tested - 4: DSO... (nested UITextMenu submenu) - 5: Stars... (nested UITextMenu submenu) + 2: Asteroids (UIObjectList, catalog "MP") + 3: NGC (UIObjectList, catalog "NGC") + 4: Messier (UIObjectList, catalog "M") ← already tested + 5: DSO... (nested UITextMenu submenu) + 6: Stars... (nested UITextMenu submenu) Key sequences from navigate_to_root_menu() (lands on Objects in root menu): R → enter Objects submenu (now at All Filtered, index 0) @@ -145,10 +146,10 @@ def test_objects_by_catalog_ngc(driver): navigate_to_root_menu(driver) # R = Objects submenu; D = By Catalog; R = enter By Catalog at Planets (0) - # DD = NGC (2); R = enter + # DDD = NGC (3); R = enter press_keys_and_validate( driver, - "RDRDDR", + "RDRDDDR", { "ui_type": "UIObjectList", "title": "NGC", @@ -158,6 +159,24 @@ def test_objects_by_catalog_ngc(driver): press_keys(driver, "ZL") # back to root +@pytest.mark.web +def test_objects_by_catalog_asteroids(driver): + """Objects > By Catalog > Asteroids opens the Asteroids object list.""" + login_to_remote(driver) + navigate_to_root_menu(driver) + + press_keys_and_validate( + driver, + "RDRDDR", + { + "ui_type": "UIObjectList", + "title": "Asteroids", + }, + ) + + press_keys(driver, "ZL") + + # --------------------------------------------------------------------------- # Objects > By Catalog > DSO... (nested submenu) # --------------------------------------------------------------------------- @@ -170,10 +189,10 @@ def test_objects_by_catalog_dso_submenu_entry(driver): navigate_to_root_menu(driver) # R = Objects submenu; D = By Catalog; R = enter By Catalog at Planets (0) - # DDDD = DSO... (4); R = enter + # DDDDD = DSO... (5); R = enter press_keys_and_validate( driver, - "RDRDDDDR", + "RDRDDDDDR", { "ui_type": "UITextMenu", "title": "DSO...", @@ -191,7 +210,7 @@ def test_objects_by_catalog_dso_first_item_is_abell(driver): press_keys_and_validate( driver, - "RDRDDDDR", + "RDRDDDDDR", { "ui_type": "UITextMenu", "title": "DSO...", @@ -212,7 +231,7 @@ def test_objects_by_catalog_dso_enter_catalog(driver): # Enter DSO..., navigate to Caldwell (DDDR), enter press_keys_and_validate( driver, - "RDRDDDDR", # enter DSO... + "RDRDDDDDR", # enter DSO... {"ui_type": "UITextMenu", "title": "DSO..."}, ) press_keys_and_validate( @@ -238,12 +257,12 @@ def test_objects_by_catalog_stars_submenu_entry(driver): login_to_remote(driver) navigate_to_root_menu(driver) - # By Catalog: ..., DSO...(4), Stars...(5) + # By Catalog: ..., DSO...(5), Stars...(6) # R = Objects submenu; D = By Catalog; R = enter By Catalog at Planets (0) - # DDDDD = Stars... (5); R = enter + # DDDDDD = Stars... (6); R = enter press_keys_and_validate( driver, - "RDRDDDDDR", + "RDRDDDDDDR", { "ui_type": "UITextMenu", "title": "Stars...", @@ -261,7 +280,7 @@ def test_objects_by_catalog_stars_first_item_is_bright_named(driver): press_keys_and_validate( driver, - "RDRDDDDDR", + "RDRDDDDDDR", { "ui_type": "UITextMenu", "title": "Stars...", From 2692406bff684a56a2acce1febffb015aad72038 Mon Sep 17 00:00:00 2001 From: Mike Rosseel Date: Thu, 16 Jul 2026 11:50:11 +0200 Subject: [PATCH 02/10] fix(web): accept comma or period decimal separator in location entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Coordinate/altitude fields on the web location add & edit forms were . In a comma-decimal browser locale the browser returns an empty .value for a period-formatted number (and vice-versa), so the client-side validation saw the field as empty and silently blocked submit — the Save button appeared active but did nothing. Integer entries (no separator) worked, decimals did not. Switch the numeric inputs to type=text inputmode=decimal, normalise both separators to a period in JS before validation and submit, and add a tolerant parse_coordinate() on the server so a stray comma yields a friendly validation message instead of a 500. Same bug class as #291 (equipment focal-length decimal crash). --- python/PiFinder/server.py | 34 +++++++++++++------ python/tests/test_server_coordinates.py | 40 ++++++++++++++++++++++ python/views/location_form.html | 20 +++++------ python/views/locations.html | 45 ++++++++++++++++--------- 4 files changed, 104 insertions(+), 35 deletions(-) create mode 100644 python/tests/test_server_coordinates.py diff --git a/python/PiFinder/server.py b/python/PiFinder/server.py index a993566d6..e5240fb19 100644 --- a/python/PiFinder/server.py +++ b/python/PiFinder/server.py @@ -48,6 +48,16 @@ class SignedIntConverter(IntegerConverter): SESSION_SECRET = str(uuid.uuid4()) +def parse_coordinate(value, field_name): + """Parse a coordinate/measurement field, accepting comma or period decimals.""" + if value is None: + raise ValueError(_("%s is required") % field_name) + try: + return float(str(value).strip().replace(",", ".")) + except ValueError: + raise ValueError(_("%s must be a number") % field_name) + + def auth_required(func): def auth_wrapper(*args, **kwargs): # check for and validate session @@ -354,11 +364,13 @@ def locations_page(): @auth_required def location_add(): try: - name = request.form.get("name").strip() - lat = float(request.form.get("latitude")) - lon = float(request.form.get("longitude")) - altitude = float(request.form.get("altitude")) - error_in_m = float(request.form.get("error_in_m", "0")) + name = (request.form.get("name") or "").strip() + lat = parse_coordinate(request.form.get("latitude"), _("Latitude")) + lon = parse_coordinate(request.form.get("longitude"), _("Longitude")) + altitude = parse_coordinate(request.form.get("altitude"), _("Altitude")) + error_in_m = parse_coordinate( + request.form.get("error_in_m", "0"), _("Error") + ) source = request.form.get("source", "Manual Entry") # Server-side validation @@ -412,11 +424,13 @@ def location_rename(location_id): if not (0 <= location_id < len(cfg.locations.locations)): raise ValueError("Invalid location ID") - name = request.form.get("name").strip() - lat = float(request.form.get("latitude")) - lon = float(request.form.get("longitude")) - altitude = float(request.form.get("altitude")) - error_in_m = float(request.form.get("error_in_m", "0")) + name = (request.form.get("name") or "").strip() + lat = parse_coordinate(request.form.get("latitude"), _("Latitude")) + lon = parse_coordinate(request.form.get("longitude"), _("Longitude")) + altitude = parse_coordinate(request.form.get("altitude"), _("Altitude")) + error_in_m = parse_coordinate( + request.form.get("error_in_m", "0"), _("Error") + ) source = request.form.get("source", "Manual Entry") # Server-side validation diff --git a/python/tests/test_server_coordinates.py b/python/tests/test_server_coordinates.py new file mode 100644 index 000000000..4b4864d83 --- /dev/null +++ b/python/tests/test_server_coordinates.py @@ -0,0 +1,40 @@ +"""Unit tests for web location coordinate parsing. + +The web location forms let users type coordinates, altitude and error. In a +comma-decimal browser locale a ```` yields an empty value +for a period-formatted number (and vice-versa), which silently blocked the +save. ``parse_coordinate`` accepts either separator so the server never sees a +stray comma; these tests lock that in and guard the missing/garbage cases that +previously raised an uncaught 500. +""" + +import pytest + +from PiFinder.server import parse_coordinate + + +@pytest.mark.unit +@pytest.mark.parametrize( + "raw, expected", + [ + ("51.3", 51.3), + ("51,3", 51.3), + ("51", 51.0), + (" -3,2 ", -3.2), + ("0", 0.0), + ], +) +def test_parse_coordinate_accepts_both_separators(raw, expected): + assert parse_coordinate(raw, "Latitude") == expected + + +@pytest.mark.unit +def test_parse_coordinate_missing_raises_value_error(): + with pytest.raises(ValueError): + parse_coordinate(None, "Latitude") + + +@pytest.mark.unit +def test_parse_coordinate_garbage_raises_value_error(): + with pytest.raises(ValueError): + parse_coordinate("not-a-number", "Longitude") diff --git a/python/views/location_form.html b/python/views/location_form.html index 4e171e6ac..9ad76b0d6 100644 --- a/python/views/location_form.html +++ b/python/views/location_form.html @@ -17,52 +17,52 @@
- +
- +
- +
- +
diff --git a/python/views/locations.html b/python/views/locations.html index ce026b92e..176adfcc2 100644 --- a/python/views/locations.html +++ b/python/views/locations.html @@ -127,52 +127,52 @@

{{ _('Edit Location') }}

- +
- +
- +
- +
@@ -206,6 +206,12 @@

{{ _('Confirm Delete') }}

{% block scripts %}