Init Commit

.gitignore

@@ -0,0 +1 @@
+.env

Dockerfile

@@ -0,0 +1,21 @@
+# Use official Python image
+FROM python:3.12-slim
+
+# Install dependencies for psql
+RUN apt-get update && apt-get install -y postgresql-client && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /app
+
+# Install dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy project files
+COPY . .
+
+# Expose port
+EXPOSE 80
+
+# Run Flask app
+CMD ["python", "main.py"]

README.md

@@ -0,0 +1 @@
+Python Flask web based app for crawling steam friends using Steam API

docker-compose.yml

@@ -0,0 +1,27 @@
+services:
+  web:
+    build: .
+    container_name: steam-crawl
+    ports:
+      - "80:80"
+    restart: no
+    env_file:
+      - .env
+    depends_on:
+      - db
+    entrypoint: ["./wait-for-db.sh", "db", "python", "main.py"]
+  db:
+    image: postgres:16
+    container_name: postgres-db
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+    ports:
+      - "5432:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    restart: always
+
+volumes:
+  pgdata:

env_example

@@ -0,0 +1,6 @@
+POSTGRES_USER=steamuser
+POSTGRES_PASSWORD=steampass
+POSTGRES_DB=steam_db
+DATABASE_URL=postgresql+psycopg2://steamuser:steampass@db:5432/steam_db
+STEAM_API_KEY=YOUR_STEAM_KEY
+FLASK_DEBUG=False

main.py

@@ -0,0 +1,31 @@
+from flask import render_template, jsonify
+from modules.db import db, User
+from modules.app import app
+from dotenv import load_dotenv
+import os
+import modules.steam_api as api
+
+load_dotenv()
+
+app.config['STEAM_API_KEY'] = os.getenv('STEAM_API_KEY')
+app.config['SQLALCHEMY_DATABASE_URI'] = os.getenv("DATABASE_URL")
+app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+app.config['DEBUG'] = os.getenv('DEBUG')
+
+with app.app_context():
+    db.init_app(app)
+    db.create_all()  # Create tables
+
+@app.route("/")
+def hello():
+    return render_template("main.html.jinja")
+
+@app.route("/user/<steam_id>")
+def user_route(steam_id):
+    user = api.get_user(steam_id)
+    if not user:
+        return jsonify({"error": "User not found"}), 404
+    return jsonify(user.to_dict())
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=80, debug=True)

modules/app.py

@@ -0,0 +1,4 @@
+from flask import Flask
+
+app = Flask(__name__)
+

modules/db.py

@@ -0,0 +1,50 @@
+from flask_sqlalchemy import SQLAlchemy
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+from datetime import datetime
+
+db = SQLAlchemy()
+
+class User(db.Model):
+    __tablename__ = "users"
+
+    steam_id: Mapped[int] = mapped_column(db.BigInteger, primary_key=True)
+    personaname: Mapped[str] = mapped_column(db.String(80))
+    profile_url: Mapped[str] = mapped_column(db.String(200))
+    avatar: Mapped[str] = mapped_column(db.String(200))
+    avatar_medium: Mapped[str] = mapped_column(db.String(200))
+    avatar_full: Mapped[str] = mapped_column(db.String(200))
+    community_visibility_state: Mapped[int] = mapped_column(db.Integer)
+    profile_state: Mapped[int] = mapped_column(db.Integer)
+    last_logoff: Mapped[int] = mapped_column(db.Integer)
+
+    def to_dict(self):
+        return {
+            "steam_id": self.steam_id,
+            "personaname": self.personaname,
+            "profile_url": self.profile_url,
+            "avatar": self.avatar,
+            "avatar_medium": self.avatar_medium,
+            "avatar_full": self.avatar_full,
+            "community_visibility_state": self.community_visibility_state,
+            "profile_state": self.profile_state,
+            "last_logoff": self.last_logoff,
+        }
+
+class APICall(db.Model):
+    __tablename__ = "api_calls"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    name: Mapped[str] = mapped_column(db.String(80), unique=True, nullable=False)
+    call_count: Mapped[int] = mapped_column(db.Integer, default=0, nullable=False)
+    first_called: Mapped[datetime] = mapped_column(db.DateTime, default=None, nullable=True)
+    last_called: Mapped[datetime] = mapped_column(db.DateTime, default=datetime.utcnow, nullable=False)
+
+    def register_call(self):
+        """Increment call count and update timestamp."""
+        if self.call_count is None:
+            self.call_count = 0
+        if self.first_called is None:
+            self.first_called = datetime.now()
+        self.call_count = self.call_count + 1
+        self.last_called = datetime.now()
+        db.session.commit()

modules/steam_api.py

@@ -0,0 +1,52 @@
+import requests
+from .app import app
+from .db import db, User, APICall
+from typing import Final
+
+API_NAME: Final = "STEAM_API"
+
+def record_api_call():
+    api_call = APICall.query.filter_by(name=API_NAME).first()
+    if not api_call:
+        api_call = APICall(name=API_NAME)
+        db.session.add(api_call)
+    api_call.register_call()
+    return api_call
+
+def get_user(steam_id):
+    # 1. Check DB
+    user = User.query.filter_by(steam_id=steam_id).first()
+    if user:
+        app.logger.debug("User fetch from DB")
+        return user
+
+    # 2. Fetch from Steam API
+    url = "https://api.steampowered.com/ISteamUser/GetPlayerSummaries/v2/"
+    params = {"key": app.config['STEAM_API_KEY'], "steamids": steam_id}
+    resp = requests.get(url, params=params)
+    api = record_api_call()
+    data = resp.json()
+    players = data.get("response", {}).get("players", [])
+    if not players:
+        app.logger.debug(f"User not found (API called: {api.call_count})")
+        return None
+
+    player = players[0]
+
+    # 3. Save to DB
+    user = User(
+        steam_id=steam_id,
+        personaname=player.get("personaname"),
+        profile_url=player.get("profileurl"),
+        avatar=player.get("avatar"),
+        avatar_medium=player.get("avatarmedium"),
+        avatar_full=player.get("avatarfull"),
+        community_visibility_state=player.get("communityvisibilitystate"),
+        profile_state=player.get("profilestate"),
+        last_logoff=player.get("lastlogoff")
+    )
+    db.session.add(user)
+    db.session.commit()
+
+    app.logger.debug(f"User fetch from API (API called: {api.call_count})")
+    return user

requirements.txt

@@ -0,0 +1,5 @@
+flask==3.1.2
+Flask-SQLAlchemy==3.0.3
+psycopg2-binary==2.9.9
+python-dotenv==1.1.1
+requests==2.31.0

templates/main.html.jinja

@@ -0,0 +1,7 @@
+<!DOCTYPE html>
+<head>
+    <title>Steam Crawl</title>
+</head>
+<body>
+    <h1>Test</h1>
+</body>

wait-for-db.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+# wait-for-db.sh
+
+set -e
+
+host="$1"
+shift
+cmd="$@"
+
+until PGPASSWORD=$POSTGRES_PASSWORD psql -h "$host" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c '\q'; do
+  echo "Waiting for database at $host..."
+  sleep 2
+done
+
+echo "Database is ready, executing command..."
+exec $cmd