add package scripts

.idea/.name

@@ -0,0 +1 @@
+energy_price_scraper

.idea/modules.xml

@@ -2,7 +2,7 @@
 <project version="4">
   <component name="ProjectModuleManager">
     <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/entsoe_scrapper.iml" filepath="$PROJECT_DIR$/.idea/entsoe_scrapper.iml" />
+      <module fileurl="file://$PROJECT_DIR$/.idea/energy_price_scraper.iml" filepath="$PROJECT_DIR$/.idea/energy_price_scraper.iml" />
     </modules>
   </component>
 </project>

README.md

@@ -0,0 +1,19 @@
+sudo apt update
+sudo apt install -y python3-venv python3-pip git tzdata
+
+# system user without shell login
+sudo useradd -r -s /usr/sbin/nologin -d /opt/energy-scrapers energy
+
+copy service to 
+/etc/systemd/system/energy-price-scrapers.service
+
+sudo install -m 0755 os/energy-price-scrapers-update.sh /usr/local/bin/energy-scrapers-update
+
+as root 
+
+systemctl daemon-reload
+systemctl enable --now energy-price-scrapers.service
+systemctl status energy-price-scrapers.service
+~~~~
+logi
+journalctl -u energy-price-scrapers.service -f

app.py

@@ -25,6 +25,7 @@ if __name__ == "__main__":
     # scraper.ingest_day(datetime.now(tz=WARSAW_TZ))
 
     last = scraper.last_entry()
+    if last is not None
 
     # scraper.ingest_day(datetime.now(tz=WARSAW_TZ) )
 

install.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# --- config ---
+REPO_URL="${REPO_URL:-<PUT_YOUR_GIT_URL_HERE>}"
+APP_DIR="/opt/energy-price-scrapers"
+
+# --- clone or update repo ---
+if [ ! -d "$APP_DIR/.git" ]; then
+  git clone "$REPO_URL" "$APP_DIR"
+else
+  git -C "$APP_DIR" fetch --all --prune
+  git -C "$APP_DIR" checkout main
+  git -C "$APP_DIR" pull --ff-only
+fi
+
+# --- venv build/refresh ---
+python3 -m venv "$APP_DIR/.venv"
+"$APP_DIR/.venv/bin/pip" install --upgrade pip setuptools wheel
+if [ -f "$APP_DIR/requirements.txt" ]; then
+  "$APP_DIR/.venv/bin/pip" install -r "$APP_DIR/requirements.txt"
+else
+  echo "requirements.txt missing; aborting."
+  exit 1
+fi
+
+chown -R energy:energy "$APP_DIR"
+echo "Install complete."

logging_setup.py

@@ -0,0 +1,32 @@
+import logging
+import logging.config
+
+class EnsureContext(logging.Filter):
+    """Always provide 'context' so the formatter never KeyErrors."""
+    def filter(self, record: logging.LogRecord) -> bool:
+        if not hasattr(record, "context"):
+            record.context = ""
+        return True
+
+def setup_logging(level: int = logging.INFO) -> None:
+    logging.config.dictConfig({
+        "version": 1,
+        "disable_existing_loggers": False,
+        "filters": {"ensure_context": {"()": EnsureContext}},
+        "formatters": {
+            "std": {
+                "format": "%(asctime)s %(levelname)s [%(name)s] %(context)s%(message)s",
+                "datefmt": "%Y-%m-%d %H:%M:%S",
+            }
+        },
+        "handlers": {
+            "console": {
+                "class": "logging.StreamHandler",
+                "level": level,
+                "formatter": "std",
+                "filters": ["ensure_context"],
+                "stream": "ext://sys.stdout",
+            }
+        },
+        "root": {"level": level, "handlers": ["console"]},
+    })

logging_utils.py

@@ -0,0 +1,65 @@
+import logging
+from contextlib import contextmanager
+
+def _fmt_ctx(ctx: dict) -> str:
+    # Build a single bracketed context block for the formatter: "[k=v a=b] "
+    if not ctx:
+        return ""
+    kv = " ".join(f"{k}={v}" for k, v in ctx.items() if v is not None)
+    return f"[{kv}] "
+
+
+class HasLogger:
+    """Mixin with explicit logger initialization. No __init__, no MRO dependency."""
+
+    def __init__(self):
+        self.log = None
+        self._log_ctx = None
+        self._base_logger = None
+
+    def init_logger(self, *, context: dict | None = None, name: str | None = None) -> None:
+        """Initialize the logger explicitly; call from your class __init__."""
+        base_name = name or self.__class__.__name__
+        self._base_logger = logging.getLogger(base_name)
+        self._log_ctx: dict = dict(context or {})
+        self.log = logging.LoggerAdapter(self._base_logger, {"context": _fmt_ctx(self._log_ctx)})
+
+    def set_logger_context(self, **context) -> None:
+        """Persistently merge new context into this instance and refresh adapter."""
+        if not hasattr(self, "_base_logger"):  # safety if init_logger was forgotten
+            self.init_logger()
+        self._log_ctx.update({k: v for k, v in context.items() if v is not None})
+        self.log = logging.LoggerAdapter(self._base_logger, {"context": _fmt_ctx(self._log_ctx)})
+
+    def child_logger(self, **extra) -> logging.LoggerAdapter:
+        """Temporary adapter with additional context (does not mutate instance context)."""
+        if not hasattr(self, "_base_logger"):
+            self.init_logger()
+        merged = self._log_ctx.copy()
+        merged.update({k: v for k, v in extra.items() if v is not None})
+        return logging.LoggerAdapter(self._base_logger, {"context": _fmt_ctx(merged)})
+
+    @contextmanager
+    def scoped_log(self, **extra):
+        """Context manager yielding a temporary adapter with extra context."""
+        yield self.child_logger(**extra)
+
+
+    #class PSE_RCEScraper(HasLogger):
+    # SIDE = "sell"
+    #
+    # def __init__(self, **kwargs):
+    #     super().__init__(logger_context={"side": self.SIDE})
+    #     self.log.info("Initialized")
+    #
+    # def fetch_day(self, day):
+    #     # Add per-call context without changing the base context
+    #     with self.scoped_log(day=day) as log:
+    #         log.info("Fetching")
+    #         # ...
+    #         log.info("Fetched %d rows", 96)
+    #
+    # def switch_side(self, side: str):
+    #     # Persistently extend instance context
+    #     self.set_logger_context(side=side)
+    #     self.log.info("Side updated")

os/energy-price-scrapers-update.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+APP_DIR="/opt/energy-price-scrapers"
+
+sudo -u energy git -C "$APP_DIR" fetch --all --prune
+sudo -u energy git -C "$APP_DIR" checkout main
+sudo -u energy git -C "$APP_DIR" pull --ff-only
+
+# upgrade deps if changed
+sudo -u energy "$APP_DIR/.venv/bin/pip" install --upgrade pip
+if [ -f "$APP_DIR/requirements.txt" ]; then
+  sudo -u energy "$APP_DIR/.venv/bin/pip" install -r "$APP_DIR/requirements.txt"
+fi
+
+sudo systemctl restart energy-price-scrapers.service
+echo "Updated & restarted."

os/energy-price-scrapers.service

@@ -0,0 +1,23 @@
+[Unit]
+Description=Energy price scrapers (APScheduler)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=energy
+Group=energy
+WorkingDirectory=/opt/energy-price-scrapers
+EnvironmentFile=/etc/energy-price-scrapers.env
+# Use the venv python to run your scheduler app
+ExecStart=/opt/energy-price-scrapers/.venv/bin/python /opt/energy-price-scrapers/app.py
+Restart=always
+RestartSec=5
+# Hardening (tune as needed)
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=full
+
+[Install]
+WantedBy=multi-user.target

requirements.txt

@@ -0,0 +1,7 @@
+psycopg[binary]>=3.1,<3.3
+pandas~=2.3.2
+requests>=2.32,<3.0
+
+apscheduler>=3.10,<4.0
+
+matplotlib~=3.10.5