initial commit

This commit is contained in:
TheOnlyMace
2026-06-18 23:45:08 +02:00
commit 7ba9687619
96 changed files with 4309 additions and 0 deletions

View File

View File

@@ -0,0 +1,67 @@
from __future__ import annotations
import asyncio
import logging
from typing import Any
from app.cache import cache_get, cache_set
from app.config import CACHE_TTL_DASHBOARD, CACHE_TTL_OBSERVATIONS, CACHE_TTL_WARNINGS
from app.services.dwd_radar import fetch_radar_index
from app.services.warnings import fetch_warnings_for_location
from app.services.weather import get_forecast, get_observations
logger = logging.getLogger(__name__)
async def get_radar_summary() -> dict[str, Any]:
redis_key = "radar:summary"
cached = await cache_get(redis_key)
if cached:
return {**cached, "cached": True}
index = await fetch_radar_index(limit=12)
payload = {
"source": index["source"],
"product": index["product"],
"count": index["count"],
"cached": False,
}
await cache_set(redis_key, payload, 120)
return payload
async def build_dashboard(lat: float, lon: float, local_only: bool = True) -> dict[str, Any]:
redis_key = f"dashboard:{round(lat, 4)}:{round(lon, 4)}:{int(local_only)}"
cached = await cache_get(redis_key)
if cached:
return {**cached, "bundle_cached": True}
results = await asyncio.gather(
get_forecast(lat, lon),
get_observations(lat, lon),
fetch_warnings_for_location(lat, lon, local_only=local_only),
get_radar_summary(),
return_exceptions=True,
)
payload: dict[str, Any] = {
"lat": lat,
"lon": lon,
"bundle_cached": False,
"errors": {},
}
names = ("forecast", "observations", "warnings", "radar")
for name, result in zip(names, results):
if isinstance(result, Exception):
logger.warning("dashboard %s failed: %s", name, result)
payload["errors"][name] = str(result)
payload[name] = None
else:
payload[name] = result
ttl = min(CACHE_TTL_DASHBOARD, CACHE_TTL_OBSERVATIONS, CACHE_TTL_WARNINGS)
if not payload["errors"]:
await cache_set(redis_key, payload, ttl)
return payload

View File

@@ -0,0 +1,43 @@
from __future__ import annotations
import re
from typing import Any
from urllib.parse import urljoin
from app.config import DWD_RADAR_BASE_URL, RADAR_PRODUCT
from app.http_client import fetch_text
RADAR_FILE_RE = re.compile(r"^[A-Za-z0-9_.-]+\.(?:tar|tar\.bz2|bz2|gz)$", re.IGNORECASE)
def parse_apache_index(html: str, suffixes: tuple[str, ...] | None = None) -> list[dict[str, Any]]:
files: list[dict[str, Any]] = []
pattern = re.compile(
r'href="(?P<href>[^"]+)"[^>]*>[^<]+</a>\s*'
r'(?P<date>\d{2}-[A-Za-z]{3}-\d{4})\s+(?P<time>\d{2}:\d{2}(?::\d{2})?)\s+(?P<size>[0-9\-]+)?',
re.IGNORECASE,
)
for match in pattern.finditer(html):
href = match.group("href")
if href.startswith("../") or href.endswith("/"):
continue
if suffixes and not href.lower().endswith(tuple(s.lower() for s in suffixes)):
continue
files.append(
{
"name": href,
"modified": f"{match.group('date')} {match.group('time')}",
"size_bytes": None if match.group("size") in (None, "-") else int(match.group("size")),
}
)
return files
async def fetch_radar_index(product: str = RADAR_PRODUCT, limit: int = 12) -> dict[str, Any]:
base_url = f"{DWD_RADAR_BASE_URL}/{product}/"
html = await fetch_text(base_url, timeout=12.0)
files = parse_apache_index(html, suffixes=(".tar", ".tar.bz2", ".bz2", ".gz"))
files = files[-limit:]
for item in files:
item["url"] = urljoin(base_url, item["name"])
return {"source": base_url, "product": product, "count": len(files), "files": files}

View File

@@ -0,0 +1,114 @@
from __future__ import annotations
import hashlib
from typing import Any
from app.config import CACHE_TTL_GEOCODE, NOMINATIM_BASE_URL, NOMINATIM_USER_AGENT
from app.http_client import get_client
from app.database import get_cached_json, set_cached_json
def _query_hash(query: str) -> str:
return hashlib.sha256(query.strip().lower().encode("utf-8")).hexdigest()
async def _nominatim_get(path: str, params: dict[str, Any]) -> Any:
headers = {"User-Agent": NOMINATIM_USER_AGENT, "Accept-Language": "de"}
client = get_client()
response = await client.get(f"{NOMINATIM_BASE_URL}{path}", params=params, headers=headers, timeout=12.0)
response.raise_for_status()
return response.json()
async def search_places(query: str, limit: int = 8) -> list[dict[str, Any]]:
if len(query.strip()) < 2:
return []
qhash = _query_hash(f"search:{query}:{limit}")
cached = await get_cached_json("geocode_cache", {"query_hash": qhash}, CACHE_TTL_GEOCODE)
if cached is not None:
return cached
raw = await _nominatim_get(
"/search",
{
"q": query,
"format": "jsonv2",
"addressdetails": 1,
"limit": limit,
"countrycodes": "de,at,ch",
},
)
results = []
for item in raw:
address = item.get("address") or {}
label = item.get("display_name", "")
place = (
address.get("city")
or address.get("town")
or address.get("village")
or address.get("municipality")
or address.get("county")
or label.split(",")[0]
)
results.append(
{
"label": label,
"place": place,
"lat": float(item["lat"]),
"lon": float(item["lon"]),
"type": item.get("type"),
"postcode": address.get("postcode"),
"county": address.get("county"),
"state": address.get("state"),
}
)
await set_cached_json(
"geocode_cache",
{"query_hash": qhash, "query_text": query},
results,
)
return results
async def reverse_geocode(lat: float, lon: float) -> dict[str, Any]:
qhash = _query_hash(f"reverse:{lat:.4f}:{lon:.4f}")
cached = await get_cached_json("geocode_cache", {"query_hash": qhash}, CACHE_TTL_GEOCODE)
if cached is not None:
return cached
raw = await _nominatim_get(
"/reverse",
{"lat": lat, "lon": lon, "format": "jsonv2", "addressdetails": 1, "zoom": 10},
)
address = raw.get("address") or {}
payload = {
"label": raw.get("display_name"),
"place": address.get("city")
or address.get("town")
or address.get("village")
or address.get("municipality")
or address.get("county"),
"county": address.get("county"),
"state": address.get("state"),
"postcode": address.get("postcode"),
"admin_names": [
value
for value in [
address.get("city"),
address.get("town"),
address.get("village"),
address.get("municipality"),
address.get("county"),
address.get("state"),
]
if value
],
}
await set_cached_json(
"geocode_cache",
{"query_hash": qhash, "query_text": f"{lat},{lon}"},
payload,
)
return payload

View File

@@ -0,0 +1,75 @@
from __future__ import annotations
import asyncio
import time
from typing import Any
from app.cache import cache_get, cache_set, redis_available
from app.config import (
BRIGHTSKY_BASE_URL,
CACHE_TTL_HEALTH_SOURCES,
DATABASE_URL,
DWD_RADAR_BASE_URL,
DWD_WMS_URL,
DWD_WARNINGS_URL,
OPEN_METEO_BASE_URL,
)
from app.database import pool_available
from app.http_client import get_client
async def _probe(name: str, url: str, timeout: float = 5.0) -> dict[str, Any]:
started = time.perf_counter()
try:
client = get_client()
response = await client.get(url, timeout=timeout)
elapsed_ms = round((time.perf_counter() - started) * 1000, 1)
ok = response.status_code < 500
return {
"name": name,
"url": url,
"status": "ok" if ok else "degraded",
"http_status": response.status_code,
"latency_ms": elapsed_ms,
}
except Exception as exc:
elapsed_ms = round((time.perf_counter() - started) * 1000, 1)
return {
"name": name,
"url": url,
"status": "error",
"error": str(exc),
"latency_ms": elapsed_ms,
}
async def check_sources() -> list[dict[str, Any]]:
cache_key = "health:sources"
cached = await cache_get(cache_key)
if cached:
return cached
probes = await asyncio.gather(
_probe("open_meteo", f"{OPEN_METEO_BASE_URL}/forecast?latitude=52.5&longitude=13.4&current=temperature_2m"),
_probe("brightsky", f"{BRIGHTSKY_BASE_URL}/current_weather?lat=52.5&lon=13.4"),
_probe("dwd_radar", f"{DWD_RADAR_BASE_URL}/rv/", timeout=4.0),
_probe("dwd_wms", f"{DWD_WMS_URL}?service=WMS&request=GetCapabilities", timeout=6.0),
_probe("dwd_warnings", DWD_WARNINGS_URL, timeout=4.0),
)
results = list(probes)
results.append(
{
"name": "postgres",
"url": DATABASE_URL.split("@")[-1],
"status": "ok" if pool_available() else "disabled",
}
)
results.append(
{
"name": "redis",
"url": "redis",
"status": "ok" if redis_available() else "disabled",
}
)
await cache_set(cache_key, results, CACHE_TTL_HEALTH_SOURCES)
return results

View File

@@ -0,0 +1,93 @@
from __future__ import annotations
import asyncio
import logging
from typing import Any
import httpx
from app.cache import cache_get, cache_set
from app.config import CACHE_TTL_GEOCODE, DWD_WMS_URL
from app.http_client import get_client
logger = logging.getLogger(__name__)
WFS_URL = f"{DWD_WMS_URL.rsplit('/wms', 1)[0]}/ows"
# Landkreise reicht fuer warnapp_landkreise/json/warnings.json
WFS_LAYERS = ("dwd:Warngebiete_Kreise",)
async def _wfs_intersects(layer: str, lat: float, lon: float) -> list[dict[str, Any]]:
params = {
"service": "WFS",
"version": "2.0.0",
"request": "GetFeature",
"typeName": layer,
"outputFormat": "application/json",
"count": 5,
"CQL_FILTER": f"INTERSECTS(SHAPE,POINT({lat} {lon}))",
}
client = get_client()
response = await client.get(WFS_URL, params=params, timeout=8.0)
response.raise_for_status()
payload = response.json()
results = []
for feature in payload.get("features", []):
props = feature.get("properties") or {}
cell_id = props.get("WARNCELLID")
if cell_id is None:
continue
results.append(
{
"warncell_id": str(cell_id),
"name": props.get("NAME"),
"short_name": props.get("KURZNAME"),
"layer": layer.split(":")[-1],
"state": props.get("BL"),
}
)
return results
def _related_cell_ids(cell_id: str) -> set[str]:
ids = {cell_id}
if len(cell_id) >= 9:
core = cell_id[1:]
ids.add(f"1{core}")
ids.add(f"9{core}")
if cell_id.startswith("9"):
ids.add(f"1{cell_id[1:]}")
elif cell_id.startswith("1"):
ids.add(f"9{cell_id[1:]}")
return ids
async def resolve_warncells(lat: float, lon: float) -> dict[str, Any]:
cache_key = f"warncell:{lat:.4f}:{lon:.4f}"
cached = await cache_get(cache_key)
if cached:
return cached
layer_results = await asyncio.gather(
*[_wfs_intersects(layer, lat, lon) for layer in WFS_LAYERS],
return_exceptions=True,
)
areas: list[dict[str, Any]] = []
for layer, result in zip(WFS_LAYERS, layer_results):
if isinstance(result, Exception):
logger.warning("WFS warncell lookup failed for %s: %s", layer, result)
continue
areas.extend(result)
cell_ids: set[str] = set()
for area in areas:
cell_ids.update(_related_cell_ids(area["warncell_id"]))
payload = {
"lat": lat,
"lon": lon,
"areas": areas,
"cell_ids": sorted(cell_ids),
}
await cache_set(cache_key, payload, CACHE_TTL_GEOCODE)
return payload

View File

@@ -0,0 +1,128 @@
from __future__ import annotations
import json
import re
from datetime import datetime, timezone
from typing import Any
from app.cache import cache_get, cache_set
from app.config import CACHE_TTL_WARNINGS, DWD_WARNINGS_URL
from app.database import set_cached_json
from app.http_client import get_client
from app.services.warncells import resolve_warncells
WARN_LEVELS = {
10: ("Vorabinformation", "info"),
20: ("Wetterwarnung", "minor"),
30: ("Markantes Wetter", "moderate"),
40: ("Unwetterwarnung", "severe"),
50: ("Extremes Unwetter", "extreme"),
60: ("Extremes Unwetter", "extreme"),
70: ("Extremes Unwetter", "extreme"),
80: ("Extremes Unwetter", "extreme"),
90: ("Extremes Unwetter", "extreme"),
}
def _flatten_warnings(payload: dict[str, Any]) -> list[dict[str, Any]]:
warnings_root = payload.get("warnings", {})
flattened: list[dict[str, Any]] = []
for cell_id, items in warnings_root.items():
if not isinstance(items, list):
continue
for item in items:
level = int(item.get("level", 0))
level_label, severity = WARN_LEVELS.get(level, (f"Level {level}", "unknown"))
flattened.append(
{
"cell_id": str(cell_id),
"state": item.get("state"),
"region_name": item.get("regionName"),
"event": item.get("event"),
"headline": item.get("headline"),
"description": (item.get("description") or "").strip(),
"instruction": (item.get("instruction") or "").strip(),
"level": level,
"level_label": level_label,
"severity": severity,
"type": item.get("type"),
"start": datetime.fromtimestamp(item["start"] / 1000, tz=timezone.utc).isoformat()
if item.get("start")
else None,
"end": datetime.fromtimestamp(item["end"] / 1000, tz=timezone.utc).isoformat()
if item.get("end")
else None,
}
)
flattened.sort(key=lambda w: (-w["level"], w.get("start") or ""))
return flattened
def _normalize_kreis_name(name: str) -> str:
value = name.lower().strip()
value = re.sub(r"^(landkreis|kreis und stadt|kreis|stadt)\s+", "", value)
value = re.sub(r"[^a-zäöüß0-9]+", " ", value)
return " ".join(value.split())
def _match_by_area_names(warning: dict[str, Any], area_names: list[str]) -> bool:
region = _normalize_kreis_name(warning.get("region_name") or "")
if not region:
return False
for area_name in area_names:
normalized = _normalize_kreis_name(area_name or "")
if not normalized or len(normalized) < 3:
continue
if region == normalized or normalized in region or region in normalized:
return True
return False
async def fetch_all_warnings() -> list[dict[str, Any]]:
cache_key = "warnings:all"
cached = await cache_get(cache_key)
if cached:
return cached
client = get_client()
response = await client.get(DWD_WARNINGS_URL, timeout=10.0)
response.raise_for_status()
text = response.text
if text.startswith("warnWetter.loadWarnings("):
text = text[text.index("(") + 1 : text.rindex(")")]
payload = json.loads(text)
warnings = _flatten_warnings(payload)
await cache_set(cache_key, warnings, CACHE_TTL_WARNINGS)
await set_cached_json("warnings_cache", {}, payload)
return warnings
async def fetch_warnings_for_location(lat: float, lon: float, local_only: bool = True) -> dict[str, Any]:
warnings = await fetch_all_warnings()
warncells = await resolve_warncells(lat, lon)
cell_ids = set(warncells.get("cell_ids", []))
area_names = [area.get("name") for area in warncells.get("areas", []) if area.get("name")]
local = [
warning
for warning in warnings
if warning["cell_id"] in cell_ids or _match_by_area_names(warning, area_names)
]
if local_only:
result_warnings = local
else:
result_warnings = warnings[:100]
return {
"lat": lat,
"lon": lon,
"count_total": len(warnings),
"count_local": len(local),
"warnings": result_warnings,
"warncells": warncells.get("areas", []),
"matched_cell_ids": sorted(cell_ids),
"local_only": local_only,
"all_warnings_truncated": not local_only and len(warnings) > 100,
}

View File

@@ -0,0 +1,66 @@
from __future__ import annotations
from typing import Any
from app.cache import cache_get as redis_get, cache_set as redis_set
from app.config import (
BRIGHTSKY_BASE_URL,
CACHE_TTL_FORECAST,
CACHE_TTL_OBSERVATIONS,
OPEN_METEO_BASE_URL,
)
from app.database import get_cached_json, set_cached_json
from app.http_client import fetch_json
FORECAST_PARAMS = {
"timezone": "Europe/Berlin",
"current": "temperature_2m,relative_humidity_2m,apparent_temperature,precipitation,rain,showers,snowfall,weather_code,cloud_cover,pressure_msl,wind_speed_10m,wind_direction_10m,wind_gusts_10m",
"hourly": "temperature_2m,relative_humidity_2m,precipitation_probability,precipitation,weather_code,cloud_cover,pressure_msl,wind_speed_10m,wind_direction_10m,wind_gusts_10m,visibility",
"daily": "weather_code,temperature_2m_max,temperature_2m_min,precipitation_sum,precipitation_probability_max,wind_speed_10m_max,wind_gusts_10m_max,sunrise,sunset",
"forecast_days": 7,
}
def _coord_key(lat: float, lon: float) -> tuple[float, float]:
return round(lat, 4), round(lon, 4)
async def get_forecast(lat: float, lon: float) -> dict[str, Any]:
lat_k, lon_k = _coord_key(lat, lon)
redis_key = f"forecast:{lat_k}:{lon_k}"
cached = await redis_get(redis_key)
if cached:
return {"source": OPEN_METEO_BASE_URL, "lat": lat, "lon": lon, "data": cached, "cached": True}
cached = await get_cached_json("forecast_cache", {"lat": lat_k, "lon": lon_k}, CACHE_TTL_FORECAST)
if cached:
await redis_set(redis_key, cached, CACHE_TTL_FORECAST)
return {"source": OPEN_METEO_BASE_URL, "lat": lat, "lon": lon, "data": cached, "cached": True}
data = await fetch_json(
f"{OPEN_METEO_BASE_URL}/forecast",
params={"latitude": lat, "longitude": lon, **FORECAST_PARAMS},
)
await set_cached_json("forecast_cache", {"lat": lat_k, "lon": lon_k}, data)
await redis_set(redis_key, data, CACHE_TTL_FORECAST)
return {"source": OPEN_METEO_BASE_URL, "lat": lat, "lon": lon, "data": data, "cached": False}
async def get_observations(lat: float, lon: float) -> dict[str, Any]:
lat_k, lon_k = _coord_key(lat, lon)
redis_key = f"observations:{lat_k}:{lon_k}"
cached = await redis_get(redis_key)
if cached:
return {"source": BRIGHTSKY_BASE_URL, "lat": lat, "lon": lon, "data": cached, "cached": True}
cached = await get_cached_json("observations_cache", {"lat": lat_k, "lon": lon_k}, CACHE_TTL_OBSERVATIONS)
if cached:
await redis_set(redis_key, cached, CACHE_TTL_OBSERVATIONS)
return {"source": BRIGHTSKY_BASE_URL, "lat": lat, "lon": lon, "data": cached, "cached": True}
data = await fetch_json(f"{BRIGHTSKY_BASE_URL}/current_weather", params={"lat": lat, "lon": lon})
await set_cached_json("observations_cache", {"lat": lat_k, "lon": lon_k}, data)
await redis_set(redis_key, data, CACHE_TTL_OBSERVATIONS)
return {"source": BRIGHTSKY_BASE_URL, "lat": lat, "lon": lon, "data": data, "cached": False}

92
api/app/services/wms.py Normal file
View File

@@ -0,0 +1,92 @@
from __future__ import annotations
import re
from datetime import datetime, timedelta, timezone
from xml.etree import ElementTree as ET
from app.http_client import get_client
from app.config import CACHE_TTL_WMS_TIMES, DWD_WMS_URL
NS = {"wms": "http://www.opengis.net/wms"}
def _parse_iso_duration_minutes(duration: str) -> int:
match = re.fullmatch(r"PT(\d+)M", duration.strip().upper())
if not match:
return 5
return int(match.group(1))
def _parse_time_dimension(value: str) -> tuple[datetime, datetime, int]:
start_raw, end_raw, step_raw = value.split("/")
start = datetime.fromisoformat(start_raw.replace("Z", "+00:00"))
end = datetime.fromisoformat(end_raw.replace("Z", "+00:00"))
step_minutes = _parse_iso_duration_minutes(step_raw)
return start, end, step_minutes
def _generate_time_steps(start: datetime, end: datetime, step_minutes: int) -> list[str]:
steps: list[str] = []
current = start
delta = timedelta(minutes=step_minutes)
while current <= end:
steps.append(current.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.000Z"))
current += delta
return steps
def _find_layer_dimension(xml_text: str, layer_name: str) -> str | None:
root = ET.fromstring(xml_text)
target = layer_name.split(":")[-1]
for layer in root.iter():
if not layer.tag.endswith("Layer"):
continue
name_el = layer.find("wms:Name", NS) or layer.find("Name")
if name_el is None or name_el.text != target:
continue
for dim in layer:
if dim.tag.endswith("Dimension") and (dim.attrib.get("name") or "").lower() == "time":
return (dim.text or "").strip()
return None
async def fetch_wms_time_steps(layer: str, minutes: int = 120) -> dict:
cache_key = f"wms:times:{layer}:{minutes}"
cached = await cache_get(cache_key)
if cached:
return cached
params = {"service": "WMS", "version": "1.3.0", "request": "GetCapabilities"}
client = get_client()
response = await client.get(DWD_WMS_URL, params=params, timeout=15.0)
response.raise_for_status()
dimension = _find_layer_dimension(response.text, layer)
if not dimension:
raise ValueError(f"No TIME dimension found for layer {layer}")
if "/" in dimension:
start, end, step_minutes = _parse_time_dimension(dimension)
all_steps = _generate_time_steps(start, end, step_minutes)
else:
all_steps = [item.strip() for item in dimension.split(",") if item.strip()]
cutoff = datetime.now(timezone.utc) - timedelta(minutes=minutes)
filtered = []
for step in all_steps:
try:
ts = datetime.fromisoformat(step.replace("Z", "+00:00"))
except ValueError:
continue
if ts >= cutoff:
filtered.append(step)
payload = {
"layer": layer,
"minutes": minutes,
"step_minutes": _parse_iso_duration_minutes("PT5M") if "/" in dimension else 5,
"count": len(filtered),
"times": filtered,
"latest": filtered[-1] if filtered else None,
}
await cache_set(cache_key, payload, CACHE_TTL_WMS_TIMES)
return payload