Legrems/opus-submitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: opus-submitter -> polylan-submitter

Browse Source
This commit is contained in:
Legrems
2026-05-09 23:25:34 +02:00
parent eb1eed852b
commit 404af4f90d
101 changed files with 14 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
polylan_submitter/submissions/__init__.py
View File
polylan_submitter/submissions/admin.py
+388
View File
@@ -0,0 +1,388 @@
from django.contrib import admin
from django.utils.html import format_html
from django.utils import timezone
from submissions.models import (
SteamAPIKey,
SteamCollection,
SteamCollectionItem,
Submission,
PuzzleResponse,
SubmissionFile,
)
@admin.register(SteamAPIKey)
class SteamAPIKeyAdmin(admin.ModelAdmin):
list_display = ["name", "masked_api_key", "is_active", "last_used", "created_at"]
list_filter = ["is_active", "created_at", "last_used"]
search_fields = ["name", "description"]
readonly_fields = ["created_at", "updated_at", "last_used", "masked_api_key"]
fieldsets = (
("Basic Information", {"fields": ("name", "description", "is_active")}),
(
"API Key",
{
"fields": ("api_key", "masked_api_key"),
"description": "Get your Steam API key from https://steamcommunity.com/dev/apikey",
},
),
(
"Metadata",
{
"fields": ("created_at", "updated_at", "last_used"),
"classes": ("collapse",),
},
),
)
def masked_api_key(self, obj):
"""Display masked API key in admin"""
if obj.api_key:
return format_html(
'<code style="background: #f8f9fa; padding: 2px 4px; border-radius: 3px;">{}</code>',
obj.masked_key,
)
return "No key set"
masked_api_key.short_description = "API Key (Masked)"
def get_queryset(self, request):
"""Only superusers can see API keys"""
qs = super().get_queryset(request)
if not request.user.is_superuser:
return qs.none()
return qs
def has_view_permission(self, request, obj=None):
"""Only superusers can view API keys"""
return request.user.is_superuser
def has_add_permission(self, request):
"""Only superusers can add API keys"""
return request.user.is_superuser
def has_change_permission(self, request, obj=None):
"""Only superusers can change API keys"""
return request.user.is_superuser
def has_delete_permission(self, request, obj=None):
"""Only superusers can delete API keys"""
return request.user.is_superuser
@admin.register(SteamCollection)
class SteamCollectionAdmin(admin.ModelAdmin):
list_display = [
"title",
"steam_id",
"author_name",
"total_items",
"current_favorites",
"last_fetched",
"is_active",
]
list_filter = ["is_active", "last_fetched", "created_at"]
search_fields = ["title", "steam_id", "author_name", "description"]
readonly_fields = ["steam_id", "created_at", "updated_at", "last_fetched"]
fieldsets = (
(
"Basic Information",
{"fields": ("steam_id", "url", "title", "description", "is_active")},
),
("Author Information", {"fields": ("author_name", "author_steam_id")}),
(
"Statistics",
{
"fields": (
"total_items",
"unique_visitors",
"current_favorites",
"total_favorites",
)
},
),
(
"Timestamps",
{
"fields": (
"steam_created_date",
"steam_updated_date",
"created_at",
"updated_at",
"last_fetched",
)
},
),
("Status", {"fields": ("fetch_error",)}),
)
@admin.register(SteamCollectionItem)
class SteamCollectionItemAdmin(admin.ModelAdmin):
list_display = [
"title",
"steam_item_id",
"collection",
"author_name",
"order_index",
]
list_filter = ["collection", "created_at"]
search_fields = ["title", "steam_item_id", "author_name", "description"]
readonly_fields = ["created_at", "updated_at"]
fieldsets = (
(
"Basic Information",
{
"fields": (
"collection",
"steam_item_id",
"title",
"description",
"order_index",
)
},
),
("Author Information", {"fields": ("author_name", "author_steam_id")}),
("Metadata", {"fields": ("tags",)}),
("Timestamps", {"fields": ("created_at", "updated_at")}),
("Points factor", {"fields": ("points_factor", "points_value")}),
)
class SubmissionFileInline(admin.TabularInline):
model = SubmissionFile
extra = 0
readonly_fields = ["file_size", "content_type", "ocr_processed", "created_at"]
fields = [
"file",
"original_filename",
"file_size",
"content_type",
"ocr_processed",
"ocr_error",
]
class PuzzleResponseInline(admin.TabularInline):
model = PuzzleResponse
extra = 0
readonly_fields = ["created_at", "updated_at"]
fields = [
"puzzle",
"puzzle_name",
"cost",
"cycles",
"area",
"needs_manual_validation",
"ocr_confidence_cost",
"ocr_confidence_cycles",
"ocr_confidence_area",
]
@admin.register(Submission)
class SubmissionAdmin(admin.ModelAdmin):
list_display = [
"id",
"user",
"total_responses",
"needs_validation",
"manual_validation_requested",
"is_validated",
"created_at",
]
list_filter = [
"is_validated",
"manual_validation_requested",
"created_at",
"updated_at",
]
search_fields = ["id", "user__username", "notes"]
readonly_fields = [
"id",
"created_at",
"updated_at",
"total_responses",
"needs_validation",
]
inlines = [PuzzleResponseInline]
fieldsets = (
("Basic Information", {"fields": ("id", "user", "notes")}),
(
"Validation",
{
"fields": (
"manual_validation_requested",
"is_validated",
"validated_by",
"validated_at",
)
},
),
(
"Statistics",
{
"fields": ("total_responses", "needs_validation"),
"classes": ("collapse",),
},
),
(
"Timestamps",
{"fields": ("created_at", "updated_at"), "classes": ("collapse",)},
),
)
actions = ["mark_as_validated"]
def mark_as_validated(self, request, queryset):
"""Mark selected submissions as validated"""
updated = 0
for submission in queryset:
if not submission.is_validated:
submission.is_validated = True
submission.validated_by = request.user
submission.validated_at = timezone.now()
submission.save()
# Also mark all responses as not needing validation
submission.responses.update(needs_manual_validation=False)
updated += 1
self.message_user(request, f"{updated} submissions marked as validated.")
mark_as_validated.short_description = "Mark selected submissions as validated"
@admin.register(PuzzleResponse)
class PuzzleResponseAdmin(admin.ModelAdmin):
list_display = [
"puzzle_name",
"submission",
"puzzle",
"cost",
"cycles",
"area",
"needs_manual_validation",
"created_at",
]
list_filter = ["needs_manual_validation", "puzzle__collection", "created_at"]
search_fields = [
"puzzle_name",
"submission__id",
"puzzle__title",
"cost",
"cycles",
"area",
]
readonly_fields = ["created_at", "updated_at"]
inlines = [SubmissionFileInline]
fieldsets = (
("Basic Information", {"fields": ("submission", "puzzle", "puzzle_name")}),
(
"OCR Data",
{
"fields": (
"cost",
"cycles",
"area",
"ocr_confidence_cost",
"ocr_confidence_cycles",
"ocr_confidence_area",
)
},
),
(
"Validation",
{
"fields": (
"needs_manual_validation",
"validated_cost",
"validated_cycles",
"validated_area",
)
},
),
(
"Timestamps",
{"fields": ("created_at", "updated_at"), "classes": ("collapse",)},
),
)
actions = ["mark_for_validation", "clear_validation_flag"]
def mark_for_validation(self, request, queryset):
"""Mark selected responses as needing validation"""
updated = queryset.update(needs_manual_validation=True)
self.message_user(request, f"{updated} responses marked for validation.")
def clear_validation_flag(self, request, queryset):
"""Clear validation flag for selected responses"""
updated = queryset.update(needs_manual_validation=False)
self.message_user(
request, f"{updated} responses cleared from validation queue."
)
mark_for_validation.short_description = "Mark as needing validation"
clear_validation_flag.short_description = "Clear validation flag"
@admin.register(SubmissionFile)
class SubmissionFileAdmin(admin.ModelAdmin):
list_display = [
"original_filename",
"response",
"file_size_display",
"content_type",
"ocr_processed",
"created_at",
]
list_filter = ["content_type", "ocr_processed", "created_at"]
search_fields = [
"original_filename",
"response__puzzle_name",
"response__submission__id",
]
readonly_fields = [
"file_size",
"content_type",
"ocr_processed",
"created_at",
"updated_at",
"file_url",
]
fieldsets = (
(
"File Information",
{
"fields": (
"file",
"original_filename",
"file_size",
"content_type",
"file_url",
)
},
),
("OCR Processing", {"fields": ("ocr_processed", "ocr_raw_data", "ocr_error")}),
("Relationships", {"fields": ("response",)}),
(
"Timestamps",
{"fields": ("created_at", "updated_at"), "classes": ("collapse",)},
),
)
def file_size_display(self, obj):
"""Display file size in human readable format"""
if obj.file_size < 1024:
return f"{obj.file_size} B"
elif obj.file_size < 1024 * 1024:
return f"{obj.file_size / 1024:.1f} KB"
else:
return f"{obj.file_size / (1024 * 1024):.1f} MB"
file_size_display.short_description = "File Size"
polylan_submitter/submissions/api.py
+293
View File
@@ -0,0 +1,293 @@
from ninja import Router, File
from ninja.files import UploadedFile
from ninja.pagination import paginate
from django.db import transaction
from django.core.files.base import ContentFile
from django.utils import timezone
from django.shortcuts import get_object_or_404
from typing import List
from polylan_submitter.submissions.utils import verify_and_validate_ocr_date_for_submission
from .models import Submission, PuzzleResponse, SubmissionFile, SteamCollectionItem
from .schemas import (
SubmissionIn,
SubmissionOut,
PuzzleResponseOut,
ValidationIn,
SteamCollectionItemOut,
)
router = Router()
@router.get("/puzzles", response=List[SteamCollectionItemOut])
def list_puzzles(request):
"""Get list of available puzzles"""
return SteamCollectionItem.objects.select_related("collection").filter(
collection__is_active=True
)
@router.get("/submissions", response=List[SubmissionOut])
@paginate
def list_submissions(request):
"""Get paginated list of submissions"""
return Submission.objects.prefetch_related(
"responses__files", "responses__puzzle"
).filter(user=request.user)
@router.get("/submissions/{submission_id}", response=SubmissionOut)
def get_submission(request, submission_id: str):
"""Get detailed submission by ID"""
return get_object_or_404(
Submission.objects.prefetch_related(
"responses__files", "responses__puzzle"
).filter(user=request.user),
id=submission_id,
)
@router.post("/submissions", response=SubmissionOut)
def create_submission(
request, data: SubmissionIn, files: List[UploadedFile] = File(...)
):
"""Create a new submission with multiple puzzle responses"""
# Validate that we have files
if not files:
return 400, {"detail": "At least one file is required"}
# Group files by puzzle (based on filename or order)
# For now, we'll assume files are provided in the same order as responses
if len(files) < len(data.responses):
return 400, {"detail": "Not enough files for all responses"}
print(data, files)
try:
with transaction.atomic():
# Check if any confidence score is below 80% to auto-request validation
auto_request_validation = any(
(
response_data.ocr_confidence_cost is not None
and response_data.ocr_confidence_cost < 0.8
)
or (
response_data.ocr_confidence_cycles is not None
and response_data.ocr_confidence_cycles < 0.8
)
or (
response_data.ocr_confidence_area is not None
and response_data.ocr_confidence_area < 0.8
)
for response_data in data.responses
)
# Create the submission
submission = Submission.objects.create(
user=request.user if request.user.is_authenticated else None,
notes=data.notes,
manual_validation_requested=data.manual_validation_requested
or auto_request_validation,
)
file_index = 0
for response_data in data.responses:
# Get the puzzle
puzzle = get_object_or_404(
SteamCollectionItem, id=response_data.puzzle_id
)
# Create the puzzle response
response = PuzzleResponse.objects.create(
submission=submission,
puzzle=puzzle,
puzzle_name=response_data.puzzle_name,
cost=response_data.cost,
cycles=response_data.cycles,
area=response_data.area,
needs_manual_validation=data.manual_validation_requested,
ocr_confidence_cost=response_data.ocr_confidence_cost,
ocr_confidence_cycles=response_data.ocr_confidence_cycles,
ocr_confidence_area=response_data.ocr_confidence_area,
**{
# Put validated if not manual validation is needed
"validated_cost": response_data.cost,
"validated_cycles": response_data.cycles,
"validated_area": response_data.area,
}
if not data.manual_validation_requested
else {},
)
# Process files for this response
# For simplicity, we'll take one file per response
# In a real implementation, you'd need better file-to-response mapping
print("FI", file_index, files)
if file_index < len(files):
uploaded_file = files[file_index]
# Validate file type
if not uploaded_file.content_type.startswith(("image/", "video/")):
return 400, {
"detail": f"Invalid file type: {uploaded_file.content_type}"
}
# Validate file size (256MB limit)
if uploaded_file.size > 256 * 1024 * 1024:
return 400, {"detail": "File too large (max 256MB)"}
# Create submission file
submission_file = SubmissionFile.objects.create(
response=response,
original_filename=uploaded_file.name,
file_size=uploaded_file.size,
content_type=uploaded_file.content_type,
)
# Save the file
submission_file.file.save(
uploaded_file.name, ContentFile(uploaded_file.read()), save=True
)
file_index += 1
# Check if OCR validation is needed
if not all(
[response_data.cost, response_data.cycles, response_data.area]
):
response.mark_for_validation("Incomplete OCR data")
# Reload with relations for response
submission = Submission.objects.prefetch_related(
"responses__files", "responses__puzzle"
).get(id=submission.id)
return submission
except Exception as e:
print(e)
return 500, {"detail": f"Error creating submission: {str(e)}"}
@router.put("/responses/{response_id}/validate", response=PuzzleResponseOut)
def validate_response(request, response_id: int, data: ValidationIn):
"""Manually validate a puzzle response"""
if not request.user.is_authenticated or not request.user.is_staff:
return 403, {"detail": "Admin access required"}
response = get_object_or_404(PuzzleResponse, id=response_id)
if data.puzzle is not None:
puzzle = get_object_or_404(SteamCollectionItem, id=data.puzzle)
response.puzzle = puzzle
# Update validated values
if data.validated_cost is not None:
response.validated_cost = data.validated_cost
if data.validated_cycles is not None:
response.validated_cycles = data.validated_cycles
if data.validated_area is not None:
response.validated_area = data.validated_area
# Mark as no longer needing validation if we have all values
if all([response.final_cost, response.final_cycles, response.final_area]):
response.needs_manual_validation = False
response.save()
return response
@router.put("/responses/{response_id}/validate/auto", response=PuzzleResponseOut)
def validate_auto(request, response_id: int):
"""Try to auto validate a puzzle response"""
if not request.user.is_authenticated or not request.user.is_staff:
return 403, {"detail": "Admin access required"}
response = get_object_or_404(PuzzleResponse, id=response_id)
for file in response.files.all():
verify_and_validate_ocr_date_for_submission(file)
return response
@router.get("/responses/needs-validation", response=List[PuzzleResponseOut])
def list_responses_needing_validation(request):
"""Get all responses that need manual validation"""
if not request.user.is_authenticated or not request.user.is_staff:
return 403, {"detail": "Admin access required"}
return (
PuzzleResponse.objects.filter(needs_manual_validation=True)
.filter(puzzle__collection__is_active=True)
.select_related("puzzle", "submission")
.prefetch_related("files")
)
@router.post("/submissions/{submission_id}/validate", response=SubmissionOut)
def validate_submission(request, submission_id: str):
"""Mark entire submission as validated"""
if not request.user.is_authenticated or not request.user.is_staff:
return 403, {"detail": "Admin access required"}
submission = get_object_or_404(Submission, id=submission_id)
submission.is_validated = True
submission.validated_by = request.user
submission.validated_at = timezone.now()
submission.save()
# Also mark all responses as not needing validation
submission.responses.update(needs_manual_validation=False)
# Reload with relations
submission = Submission.objects.prefetch_related(
"responses__files", "responses__puzzle"
).get(id=submission.id)
return submission
@router.delete("/submissions/{submission_id}")
def delete_submission(request, submission_id: str):
"""Delete a submission (admin only)"""
if not request.user.is_authenticated or not request.user.is_staff:
return 403, {"detail": "Admin access required"}
submission = get_object_or_404(Submission, id=submission_id)
submission.delete()
return {"detail": "Submission deleted successfully"}
@router.get("/stats")
def get_stats(request):
"""Get submission statistics"""
total_submissions = Submission.objects.count()
total_responses = PuzzleResponse.objects.count()
needs_validation = PuzzleResponse.objects.filter(
needs_manual_validation=True
).count()
validated_submissions = Submission.objects.filter(is_validated=True).count()
return {
"total_submissions": total_submissions,
"total_responses": total_responses,
"needs_validation": needs_validation,
"validated_submissions": validated_submissions,
"validation_rate": (total_responses - needs_validation) / total_responses
if total_responses
else 0,
}
polylan_submitter/submissions/apps.py
+6
View File
@@ -0,0 +1,6 @@
from django.apps import AppConfig
class SubmissionsConfig(AppConfig):
default_auto_field = "django.db.models.BigAutoField"
name = "submissions"
polylan_submitter/submissions/management/__init__.py
View File
polylan_submitter/submissions/management/commands/__init__.py
View File
polylan_submitter/submissions/management/commands/fetch_steam_collection.py
+99
View File
@@ -0,0 +1,99 @@
"""
Django management command to fetch Steam Workshop collections
"""
from django.core.management.base import BaseCommand, CommandError
from submissions.utils import create_or_update_collection
from submissions.models import SteamAPIKey, SteamCollection
class Command(BaseCommand):
help = "Fetch Steam Workshop collection data and save to database"
def add_arguments(self, parser):
parser.add_argument("url", type=str, help="Steam Workshop collection URL")
parser.add_argument(
"--force",
action="store_true",
help="Force refetch even if collection already exists",
)
def handle(self, *args, **options):
url = options["url"]
force = options["force"]
self.stdout.write(f"Fetching Steam collection from: {url}")
api_key = SteamAPIKey.objects.filter(is_active=True).first()
if not api_key:
self.stderr.write("No API key defined! Aborting...")
return
self.stdout.write(f"Using api key: {api_key}")
try:
# Check if collection already exists
from submissions.utils import SteamCollectionFetcher
fetcher = SteamCollectionFetcher(api_key.api_key)
collection_id = fetcher.extract_collection_id(url)
if collection_id and not force:
existing = SteamCollection.objects.filter(
steam_id=collection_id
).first()
if existing:
self.stdout.write(
self.style.WARNING(
f"Collection {collection_id} already exists (ID: {existing.id}). "
"Use --force to refetch."
)
)
return
# Fetch and create/update collection
collection, created = create_or_update_collection(url)
if created:
self.stdout.write(
self.style.SUCCESS(
f"Successfully created collection: {collection.title} (ID: {collection.id})"
)
)
else:
self.stdout.write(
self.style.SUCCESS(
f"Successfully updated collection: {collection.title} (ID: {collection.id})"
)
)
# Display collection info
self.stdout.write("\nCollection Details:")
self.stdout.write(f" Steam ID: {collection.steam_id}")
self.stdout.write(f" Title: {collection.title}")
self.stdout.write(f" Author: {collection.author_name or 'Unknown'}")
self.stdout.write(
f" Description: {collection.description[:100]}{'...' if len(collection.description) > 100 else ''}"
)
self.stdout.write(f" Total Items: {collection.total_items}")
self.stdout.write(f" Unique Visitors: {collection.unique_visitors}")
self.stdout.write(f" Current Favorites: {collection.current_favorites}")
self.stdout.write(f" Total Favorites: {collection.total_favorites}")
if collection.items.exists():
self.stdout.write(f"\nCollection Items ({collection.items.count()}):")
for item in collection.items.all()[:10]: # Show first 10 items
self.stdout.write(
f" - {item.title} (Steam ID: {item.steam_item_id})"
)
if collection.items.count() > 10:
self.stdout.write(
f" ... and {collection.items.count() - 10} more items"
)
else:
self.stdout.write("\nNo items found in collection.")
except Exception as e:
raise CommandError(f"Failed to fetch collection: {e}")
polylan_submitter/submissions/management/commands/recheck_ocr_validation.py
+20
View File
@@ -0,0 +1,20 @@
from django.core.management.base import BaseCommand
from submissions.utils import verify_and_validate_ocr_date_for_submission
from submissions.models import SubmissionFile
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument("--force", action="store_true", help="Force redo the OCR")
def handle(self, *args, **options):
for file in SubmissionFile.objects.prefetch_related(
"response__puzzle",
"response__submission__user",
):
if not file.response.needs_manual_validation and not options.get(
"force", False
):
continue
verify_and_validate_ocr_date_for_submission(file)
polylan_submitter/submissions/migrations/0001_initial.py
+219
View File
@@ -0,0 +1,219 @@
# Generated by Django 5.2.7 on 2025-10-29 00:11
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="Collection",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("url", models.URLField()),
],
),
migrations.CreateModel(
name="SteamCollection",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"steam_id",
models.CharField(
help_text="Steam collection ID from URL",
max_length=50,
unique=True,
),
),
(
"url",
models.URLField(help_text="Full Steam Workshop collection URL"),
),
(
"title",
models.CharField(
blank=True, help_text="Collection title", max_length=255
),
),
(
"description",
models.TextField(blank=True, help_text="Collection description"),
),
(
"author_name",
models.CharField(
blank=True,
help_text="Steam username of collection creator",
max_length=100,
),
),
(
"author_steam_id",
models.CharField(
blank=True,
help_text="Steam ID of collection creator",
max_length=50,
),
),
(
"total_items",
models.PositiveIntegerField(
default=0, help_text="Number of items in collection"
),
),
(
"unique_visitors",
models.PositiveIntegerField(
default=0, help_text="Number of unique visitors"
),
),
(
"current_favorites",
models.PositiveIntegerField(
default=0, help_text="Current number of favorites"
),
),
(
"total_favorites",
models.PositiveIntegerField(
default=0, help_text="Total unique favorites"
),
),
(
"steam_created_date",
models.DateTimeField(
blank=True,
help_text="When collection was created on Steam",
null=True,
),
),
(
"steam_updated_date",
models.DateTimeField(
blank=True,
help_text="When collection was last updated on Steam",
null=True,
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"last_fetched",
models.DateTimeField(
blank=True,
help_text="When data was last fetched from Steam",
null=True,
),
),
(
"is_active",
models.BooleanField(
default=True,
help_text="Whether this collection is actively tracked",
),
),
(
"fetch_error",
models.TextField(
blank=True,
help_text="Last error encountered when fetching data",
),
),
],
options={
"verbose_name": "Steam Collection",
"verbose_name_plural": "Steam Collections",
"ordering": ["-created_at"],
},
),
migrations.CreateModel(
name="SteamCollectionItem",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"steam_item_id",
models.CharField(help_text="Steam Workshop item ID", max_length=50),
),
(
"title",
models.CharField(
blank=True, help_text="Item title", max_length=255
),
),
(
"author_name",
models.CharField(
blank=True,
help_text="Steam username of item creator",
max_length=100,
),
),
(
"author_steam_id",
models.CharField(
blank=True, help_text="Steam ID of item creator", max_length=50
),
),
(
"description",
models.TextField(blank=True, help_text="Item description"),
),
(
"tags",
models.JSONField(
blank=True, default=list, help_text="Item tags as JSON array"
),
),
(
"order_index",
models.PositiveIntegerField(
default=0, help_text="Order of item in collection"
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"collection",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="items",
to="submissions.steamcollection",
),
),
],
options={
"verbose_name": "Steam Collection Item",
"verbose_name_plural": "Steam Collection Items",
"ordering": ["collection", "order_index"],
"unique_together": {("collection", "steam_item_id")},
},
),
]
polylan_submitter/submissions/migrations/0002_delete_collection.py
+15
View File
@@ -0,0 +1,15 @@
# Generated by Django 5.2.7 on 2025-10-29 00:12
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("submissions", "0001_initial"),
]
operations = [
migrations.DeleteModel(
name="Collection",
),
]
polylan_submitter/submissions/migrations/0003_steamapikey.py
+69
View File
@@ -0,0 +1,69 @@
# Generated by Django 5.2.7 on 2025-10-29 00:19
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("submissions", "0002_delete_collection"),
]
operations = [
migrations.CreateModel(
name="SteamAPIKey",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"name",
models.CharField(
help_text="Descriptive name for this API key (e.g., 'Production Key', 'Development Key')",
max_length=100,
unique=True,
),
),
(
"api_key",
models.CharField(
help_text="Steam Web API key from https://steamcommunity.com/dev/apikey",
max_length=64,
),
),
(
"is_active",
models.BooleanField(
default=True, help_text="Whether this API key should be used"
),
),
(
"description",
models.TextField(
blank=True,
help_text="Optional description or notes about this API key",
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"last_used",
models.DateTimeField(
blank=True,
help_text="When this API key was last used",
null=True,
),
),
],
options={
"verbose_name": "Steam API Key",
"verbose_name_plural": "Steam API Keys",
"ordering": ["-is_active", "name"],
},
),
]
polylan_submitter/submissions/migrations/0004_submission_puzzleresponse_submissionfile.py
+252
View File
@@ -0,0 +1,252 @@
# Generated by Django 5.2.7 on 2025-10-29 01:32
import django.db.models.deletion
import submissions.models
import uuid
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("submissions", "0003_steamapikey"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name="Submission",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
(
"notes",
models.TextField(
blank=True, help_text="Optional notes about the submission"
),
),
(
"is_validated",
models.BooleanField(
default=False,
help_text="Whether this submission has been manually validated",
),
),
(
"validated_at",
models.DateTimeField(
blank=True,
help_text="When this submission was validated",
null=True,
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"user",
models.ForeignKey(
blank=True,
help_text="User who made the submission (null for anonymous)",
null=True,
on_delete=django.db.models.deletion.CASCADE,
to=settings.AUTH_USER_MODEL,
),
),
(
"validated_by",
models.ForeignKey(
blank=True,
help_text="Admin user who validated this submission",
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="validated_submissions",
to=settings.AUTH_USER_MODEL,
),
),
],
options={
"verbose_name": "Submission",
"verbose_name_plural": "Submissions",
"ordering": ["-created_at"],
},
),
migrations.CreateModel(
name="PuzzleResponse",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"puzzle_name",
models.CharField(
help_text="Puzzle name as detected by OCR", max_length=255
),
),
(
"cost",
models.CharField(
blank=True, help_text="Cost value from OCR", max_length=20
),
),
(
"cycles",
models.CharField(
blank=True, help_text="Cycles value from OCR", max_length=20
),
),
(
"area",
models.CharField(
blank=True, help_text="Area value from OCR", max_length=20
),
),
(
"needs_manual_validation",
models.BooleanField(
default=False,
help_text="Whether OCR failed and manual validation is needed",
),
),
(
"ocr_confidence_score",
models.FloatField(
blank=True,
help_text="OCR confidence score (0.0 to 1.0)",
null=True,
),
),
(
"validated_cost",
models.CharField(
blank=True,
help_text="Manually validated cost value",
max_length=20,
),
),
(
"validated_cycles",
models.CharField(
blank=True,
help_text="Manually validated cycles value",
max_length=20,
),
),
(
"validated_area",
models.CharField(
blank=True,
help_text="Manually validated area value",
max_length=20,
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"puzzle",
models.ForeignKey(
help_text="The puzzle this response is for",
on_delete=django.db.models.deletion.CASCADE,
related_name="responses",
to="submissions.steamcollectionitem",
),
),
(
"submission",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="responses",
to="submissions.submission",
),
),
],
options={
"verbose_name": "Puzzle Response",
"verbose_name_plural": "Puzzle Responses",
"ordering": ["submission", "puzzle__order_index"],
"unique_together": {("submission", "puzzle")},
},
),
migrations.CreateModel(
name="SubmissionFile",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"file",
models.FileField(
help_text="Uploaded file (image/gif)",
upload_to=submissions.models.submission_file_upload_path,
),
),
(
"original_filename",
models.CharField(
help_text="Original filename as uploaded by user",
max_length=255,
),
),
(
"file_size",
models.PositiveIntegerField(help_text="File size in bytes"),
),
(
"content_type",
models.CharField(help_text="MIME type of the file", max_length=100),
),
(
"ocr_processed",
models.BooleanField(
default=False,
help_text="Whether OCR has been processed for this file",
),
),
(
"ocr_raw_data",
models.JSONField(
blank=True, help_text="Raw OCR data as JSON", null=True
),
),
(
"ocr_error",
models.TextField(
blank=True, help_text="OCR processing error message"
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"response",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="files",
to="submissions.puzzleresponse",
),
),
],
options={
"verbose_name": "Submission File",
"verbose_name_plural": "Submission Files",
"ordering": ["response", "created_at"],
},
),
]
polylan_submitter/submissions/migrations/0005_alter_submission_notes.py
+19
View File
@@ -0,0 +1,19 @@
# Generated by Django 5.2.7 on 2025-10-29 01:44
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("submissions", "0004_submission_puzzleresponse_submissionfile"),
]
operations = [
migrations.AlterField(
model_name="submission",
name="notes",
field=models.TextField(
blank=True, help_text="Optional notes about the submission", null=True
),
),
]
polylan_submitter/submissions/migrations/0006_remove_puzzleresponse_ocr_confidence_score_and_more.py
+43
View File
@@ -0,0 +1,43 @@
# Generated by Django 5.2.7 on 2025-10-30 10:46
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("submissions", "0005_alter_submission_notes"),
]
operations = [
migrations.RemoveField(
model_name="puzzleresponse",
name="ocr_confidence_score",
),
migrations.AddField(
model_name="puzzleresponse",
name="ocr_confidence_area",
field=models.FloatField(
blank=True,
help_text="OCR confidence score for area (0.0 to 1.0)",
null=True,
),
),
migrations.AddField(
model_name="puzzleresponse",
name="ocr_confidence_cost",
field=models.FloatField(
blank=True,
help_text="OCR confidence score for cost (0.0 to 1.0)",
null=True,
),
),
migrations.AddField(
model_name="puzzleresponse",
name="ocr_confidence_cycles",
field=models.FloatField(
blank=True,
help_text="OCR confidence score for cycles (0.0 to 1.0)",
null=True,
),
),
]
polylan_submitter/submissions/migrations/0007_submission_manual_validation_requested.py
+20
View File
@@ -0,0 +1,20 @@
# Generated by Django 5.2.7 on 2025-10-30 11:05
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("submissions", "0006_remove_puzzleresponse_ocr_confidence_score_and_more"),
]
operations = [
migrations.AddField(
model_name="submission",
name="manual_validation_requested",
field=models.BooleanField(
default=False,
help_text="Whether the user specifically requested manual validation",
),
),
]
polylan_submitter/submissions/migrations/0008_alter_puzzleresponse_unique_together.py
+16
View File
@@ -0,0 +1,16 @@
# Generated by Django 5.2.7 on 2025-10-30 20:39
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("submissions", "0007_submission_manual_validation_requested"),
]
operations = [
migrations.AlterUniqueTogether(
name="puzzleresponse",
unique_together=set(),
),
]
polylan_submitter/submissions/migrations/0009_steamcollectionitem_points_factor.py
+23
View File
@@ -0,0 +1,23 @@
# Generated by Django 5.2.7 on 2025-11-23 22:33
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("animations", "0001_initial"),
("submissions", "0008_alter_puzzleresponse_unique_together"),
]
operations = [
migrations.AddField(
model_name="steamcollectionitem",
name="points_factor",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="animations.puzzlepointsfactor",
),
),
]
polylan_submitter/submissions/migrations/0010_alter_puzzleresponse_validated_area_and_more.py
+33
View File
@@ -0,0 +1,33 @@
# Generated by Django 5.2.7 on 2025-11-23 23:10
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("submissions", "0009_steamcollectionitem_points_factor"),
]
operations = [
migrations.AlterField(
model_name="puzzleresponse",
name="validated_area",
field=models.IntegerField(
blank=True, help_text="Manually validated area value"
),
),
migrations.AlterField(
model_name="puzzleresponse",
name="validated_cost",
field=models.IntegerField(
blank=True, help_text="Manually validated cost value"
),
),
migrations.AlterField(
model_name="puzzleresponse",
name="validated_cycles",
field=models.IntegerField(
blank=True, help_text="Manually validated cycles value"
),
),
]
polylan_submitter/submissions/migrations/0011_alter_puzzleresponse_area_alter_puzzleresponse_cost_and_more.py
+27
View File
@@ -0,0 +1,27 @@
# Generated by Django 5.2.7 on 2025-11-23 23:13
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("submissions", "0010_alter_puzzleresponse_validated_area_and_more"),
]
operations = [
migrations.AlterField(
model_name="puzzleresponse",
name="area",
field=models.IntegerField(blank=True, help_text="Area value from OCR"),
),
migrations.AlterField(
model_name="puzzleresponse",
name="cost",
field=models.IntegerField(blank=True, help_text="Cost value from OCR"),
),
migrations.AlterField(
model_name="puzzleresponse",
name="cycles",
field=models.IntegerField(blank=True, help_text="Cycles value from OCR"),
),
]
polylan_submitter/submissions/migrations/0012_alter_puzzleresponse_validated_area_and_more.py
+36
View File
@@ -0,0 +1,36 @@
# Generated by Django 5.2.7 on 2025-11-23 23:23
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
(
"submissions",
"0011_alter_puzzleresponse_area_alter_puzzleresponse_cost_and_more",
),
]
operations = [
migrations.AlterField(
model_name="puzzleresponse",
name="validated_area",
field=models.IntegerField(
blank=True, help_text="Manually validated area value", null=True
),
),
migrations.AlterField(
model_name="puzzleresponse",
name="validated_cost",
field=models.IntegerField(
blank=True, help_text="Manually validated cost value", null=True
),
),
migrations.AlterField(
model_name="puzzleresponse",
name="validated_cycles",
field=models.IntegerField(
blank=True, help_text="Manually validated cycles value", null=True
),
),
]
polylan_submitter/submissions/migrations/0013_steamcollectionitem_points_value.py
+23
View File
@@ -0,0 +1,23 @@
# Generated by Django 5.2.7 on 2025-11-24 01:00
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("animations", "0002_puzzlepointsvalue"),
("submissions", "0012_alter_puzzleresponse_validated_area_and_more"),
]
operations = [
migrations.AddField(
model_name="steamcollectionitem",
name="points_value",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="animations.puzzlepointsvalue",
),
),
]
polylan_submitter/submissions/migrations/__init__.py
View File
polylan_submitter/submissions/models.py
+495
View File
@@ -0,0 +1,495 @@
from typing import Self
from django.db import models
from django.contrib.auth import get_user_model
from django.core.exceptions import ValidationError
import uuid
from django.db.models.expressions import Window
from django.db.models.functions import Cast, Rank, RowNumber
from django.db.models.query import F
User = get_user_model()
class JsonIndex(models.Func):
function = ""
template = "%(json_field)s -> (%(index)s::int)"
def __init__(self, json_field, index_expression, **extra):
super().__init__(json_field, index_expression, **extra)
self.output_field = models.IntegerField()
def resolve_expression(
self, query, allow_joins=True, reuse=None, summarize=False, for_save=False
):
# Resolve both expressions in the query context to ensure joins are set up correctly
clone = self.copy()
clone.source_expressions = [
expr.resolve_expression(query, allow_joins, reuse, summarize, for_save)
for expr in self.source_expressions
]
return clone
def as_sql(self, compiler, connection):
json_sql, json_params = compiler.compile(self.source_expressions[0])
idx_sql, idx_params = compiler.compile(self.source_expressions[1])
sql = self.template % {"json_field": json_sql, "index": idx_sql}
params = json_params + idx_params
return sql, params
class SteamAPIKey(models.Model):
"""Model to store Steam API key configuration - Admin only"""
name = models.CharField(
max_length=100,
unique=True,
help_text="Descriptive name for this API key (e.g., 'Production Key', 'Development Key')",
)
api_key = models.CharField(
max_length=64,
help_text="Steam Web API key from https://steamcommunity.com/dev/apikey",
)
is_active = models.BooleanField(
default=True, help_text="Whether this API key should be used"
)
description = models.TextField(
blank=True, help_text="Optional description or notes about this API key"
)
# Metadata
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
last_used = models.DateTimeField(
null=True, blank=True, help_text="When this API key was last used"
)
class Meta:
verbose_name = "Steam API Key"
verbose_name_plural = "Steam API Keys"
ordering = ["-is_active", "name"]
def __str__(self):
status = "Active" if self.is_active else "Inactive"
return f"{self.name} ({status})"
def clean(self):
"""Validate the API key format"""
if self.api_key:
# Steam API keys are typically 32 characters of hexadecimal
if len(self.api_key) != 32:
raise ValidationError("Steam API key should be 32 characters long")
# Check if it's hexadecimal
try:
int(self.api_key, 16)
except ValueError:
raise ValidationError(
"Steam API key should contain only hexadecimal characters (0-9, A-F)"
)
def save(self, *args, **kwargs):
self.full_clean()
super().save(*args, **kwargs)
@classmethod
def get_active_key(cls):
"""Get the currently active API key"""
return cls.objects.filter(is_active=True).first()
@property
def masked_key(self):
"""Return a masked version of the API key for display"""
if not self.api_key:
return ""
return f"{self.api_key[:8]}{'*' * 16}{self.api_key[-8:]}"
class SteamCollection(models.Model):
"""Model representing a Steam Workshop collection"""
# Basic collection info
steam_id = models.CharField(
max_length=50, unique=True, help_text="Steam collection ID from URL"
)
url = models.URLField(help_text="Full Steam Workshop collection URL")
title = models.CharField(max_length=255, blank=True, help_text="Collection title")
description = models.TextField(blank=True, help_text="Collection description")
# Author information
author_name = models.CharField(
max_length=100, blank=True, help_text="Steam username of collection creator"
)
author_steam_id = models.CharField(
max_length=50, blank=True, help_text="Steam ID of collection creator"
)
# Collection metadata
total_items = models.PositiveIntegerField(
default=0, help_text="Number of items in collection"
)
unique_visitors = models.PositiveIntegerField(
default=0, help_text="Number of unique visitors"
)
current_favorites = models.PositiveIntegerField(
default=0, help_text="Current number of favorites"
)
total_favorites = models.PositiveIntegerField(
default=0, help_text="Total unique favorites"
)
# Timestamps
steam_created_date = models.DateTimeField(
null=True, blank=True, help_text="When collection was created on Steam"
)
steam_updated_date = models.DateTimeField(
null=True, blank=True, help_text="When collection was last updated on Steam"
)
# Local tracking
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
last_fetched = models.DateTimeField(
null=True, blank=True, help_text="When data was last fetched from Steam"
)
# Status
is_active = models.BooleanField(
default=True, help_text="Whether this collection is actively tracked"
)
fetch_error = models.TextField(
blank=True, help_text="Last error encountered when fetching data"
)
class Meta:
ordering = ["-created_at"]
verbose_name = "Steam Collection"
verbose_name_plural = "Steam Collections"
def __str__(self):
return f"{self.title or f'Collection {self.steam_id}'}"
@property
def steam_url(self):
"""Generate the Steam Workshop URL from steam_id"""
return f"https://steamcommunity.com/workshop/filedetails/?id={self.steam_id}"
class SteamCollectionItem(models.Model):
"""Model representing individual items within a Steam collection"""
# Relationships
collection = models.ForeignKey(
SteamCollection, on_delete=models.CASCADE, related_name="items"
)
# Item identification
steam_item_id = models.CharField(max_length=50, help_text="Steam Workshop item ID")
title = models.CharField(max_length=255, blank=True, help_text="Item title")
# Author information
author_name = models.CharField(
max_length=100, blank=True, help_text="Steam username of item creator"
)
author_steam_id = models.CharField(
max_length=50, blank=True, help_text="Steam ID of item creator"
)
# Item metadata
description = models.TextField(blank=True, help_text="Item description")
tags = models.JSONField(
default=list, blank=True, help_text="Item tags as JSON array"
)
# Position in collection
order_index = models.PositiveIntegerField(
default=0, help_text="Order of item in collection"
)
# Timestamps
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
# Puzzle points
points_factor = models.ForeignKey(
"animations.PuzzlePointsFactor",
null=True,
on_delete=models.SET_NULL,
)
points_value = models.ForeignKey(
"animations.PuzzlePointsValue",
null=True,
on_delete=models.SET_NULL,
)
class Meta:
ordering = ["collection", "order_index"]
unique_together = ["collection", "steam_item_id"]
verbose_name = "Steam Collection Item"
verbose_name_plural = "Steam Collection Items"
def __str__(self):
return f"{self.title or f'Item {self.steam_item_id}'} (in {self.collection})"
@property
def steam_url(self):
"""Generate the Steam Workshop URL for this item"""
return (
f"https://steamcommunity.com/workshop/filedetails/?id={self.steam_item_id}"
)
def submission_file_upload_path(instance, filename):
"""Generate upload path for submission files"""
# Create path: submissions/{submission_id}/{uuid}_{filename}
ext = filename.split(".")[-1] if "." in filename else ""
new_filename = f"{uuid.uuid4()}_{filename}" if ext else str(uuid.uuid4())
return f"submissions/{instance.response.submission.id}/{new_filename}"
class Submission(models.Model):
"""Model representing a submission containing multiple puzzle responses"""
# Identification
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
# User information (optional for anonymous submissions)
user = models.ForeignKey(
User,
on_delete=models.CASCADE,
null=True,
blank=True,
help_text="User who made the submission (null for anonymous)",
)
# Submission metadata
notes = models.TextField(
null=True,
blank=True,
help_text="Optional notes about the submission",
)
# Status tracking
is_validated = models.BooleanField(
default=False, help_text="Whether this submission has been manually validated"
)
validated_by = models.ForeignKey(
User,
on_delete=models.SET_NULL,
null=True,
blank=True,
related_name="validated_submissions",
help_text="Admin user who validated this submission",
)
validated_at = models.DateTimeField(
null=True, blank=True, help_text="When this submission was validated"
)
# Manual validation request
manual_validation_requested = models.BooleanField(
default=False,
help_text="Whether the user specifically requested manual validation",
)
# Timestamps
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
ordering = ["-created_at"]
verbose_name = "Submission"
verbose_name_plural = "Submissions"
def __str__(self):
user_info = f"by {self.user.username}" if self.user else "anonymous"
return f"Submission {self.id} {user_info}"
@property
def total_responses(self):
"""Get total number of puzzle responses in this submission"""
return self.responses.count()
@property
def needs_validation(self):
"""Check if any response needs manual validation"""
return self.responses.filter(needs_manual_validation=True).exists()
class PuzzleResponseQuerySet(models.QuerySet):
def annotate_rank_points(self) -> Self:
return (
self.annotate(
points=F("puzzle__points_factor__cost") * F("validated_cost")
+ F("puzzle__points_factor__cycles") * F("validated_cycles")
+ F("puzzle__points_factor__area") * F("validated_area")
)
.annotate(
user_response_rank=Window(
expression=RowNumber(),
partition_by=[F("puzzle"), F("submission__user")],
order_by=F("points").asc(),
)
)
# .filter(user_response_rank=1)
.annotate(
puzzle_user_rank=Window(
expression=Rank(),
partition_by=[F("puzzle")],
order_by=F("points").asc(),
)
)
.annotate(
rank_points=Cast(
JsonIndex(
F("puzzle__points_value__points"),
Cast(F("puzzle_user_rank") - 1, models.IntegerField()),
),
models.IntegerField(),
)
)
)
def filter_user_best_response(self) -> Self:
return self.annotate_rank_points().filter(user_response_rank=1)
class PuzzleResponseManager(models.Manager.from_queryset(PuzzleResponseQuerySet)):
pass
class PuzzleResponse(models.Model):
"""Model representing a response/solution for a specific puzzle"""
# Relationships
submission = models.ForeignKey(
Submission, on_delete=models.CASCADE, related_name="responses"
)
puzzle = models.ForeignKey(
SteamCollectionItem,
on_delete=models.CASCADE,
related_name="responses",
help_text="The puzzle this response is for",
)
# OCR extracted data
puzzle_name = models.CharField(
max_length=255, help_text="Puzzle name as detected by OCR"
)
cost = models.IntegerField(blank=True, help_text="Cost value from OCR")
cycles = models.IntegerField(blank=True, help_text="Cycles value from OCR")
area = models.IntegerField(blank=True, help_text="Area value from OCR")
# Validation flags
needs_manual_validation = models.BooleanField(
default=False, help_text="Whether OCR failed and manual validation is needed"
)
ocr_confidence_cost = models.FloatField(
null=True, blank=True, help_text="OCR confidence score for cost (0.0 to 1.0)"
)
ocr_confidence_cycles = models.FloatField(
null=True, blank=True, help_text="OCR confidence score for cycles (0.0 to 1.0)"
)
ocr_confidence_area = models.FloatField(
null=True, blank=True, help_text="OCR confidence score for area (0.0 to 1.0)"
)
# Manual validation overrides
validated_cost = models.IntegerField(
null=True, blank=True, help_text="Manually validated cost value"
)
validated_cycles = models.IntegerField(
null=True, blank=True, help_text="Manually validated cycles value"
)
validated_area = models.IntegerField(
null=True, blank=True, help_text="Manually validated area value"
)
# Timestamps
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
objects = PuzzleResponseManager()
class Meta:
ordering = ["submission", "puzzle__order_index"]
verbose_name = "Puzzle Response"
verbose_name_plural = "Puzzle Responses"
_base_manager_name = "objects"
def __str__(self):
return f"Response for {self.puzzle_name} in {self.submission}"
@property
def final_cost(self):
"""Get the final cost value (validated if available, otherwise OCR)"""
return self.validated_cost or self.cost
@property
def final_cycles(self):
"""Get the final cycles value (validated if available, otherwise OCR)"""
return self.validated_cycles or self.cycles
@property
def final_area(self):
"""Get the final area value (validated if available, otherwise OCR)"""
return self.validated_area or self.area
def mark_for_validation(self, reason="OCR failed"):
"""Mark this response as needing manual validation"""
self.needs_manual_validation = True
self.save(update_fields=["needs_manual_validation"])
class SubmissionFile(models.Model):
"""Model representing files uploaded with a puzzle response"""
# Relationships
response = models.ForeignKey(
PuzzleResponse, on_delete=models.CASCADE, related_name="files"
)
# File information
file = models.FileField(
upload_to=submission_file_upload_path, help_text="Uploaded file (image/gif)"
)
original_filename = models.CharField(
max_length=255, help_text="Original filename as uploaded by user"
)
file_size = models.PositiveIntegerField(help_text="File size in bytes")
content_type = models.CharField(max_length=100, help_text="MIME type of the file")
# OCR metadata
ocr_processed = models.BooleanField(
default=False, help_text="Whether OCR has been processed for this file"
)
ocr_raw_data = models.JSONField(
null=True, blank=True, help_text="Raw OCR data as JSON"
)
ocr_error = models.TextField(blank=True, help_text="OCR processing error message")
# Timestamps
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
ordering = ["response", "created_at"]
verbose_name = "Submission File"
verbose_name_plural = "Submission Files"
def __str__(self):
return f"{self.original_filename} for {self.response}"
@property
def file_url(self):
"""Get the URL for the uploaded file"""
if self.file:
return self.file.url
return None
def save(self, *args, **kwargs):
# Set file metadata on save
if self.file and not self.file_size:
self.file_size = self.file.size
super().save(*args, **kwargs)
polylan_submitter/submissions/schemas.py
+182
View File
@@ -0,0 +1,182 @@
from ninja import Schema, ModelSchema
from typing import List, Optional
from datetime import datetime
from uuid import UUID
from .models import Submission, PuzzleResponse, SubmissionFile, SteamCollectionItem
# Input Schemas
class SubmissionFileIn(Schema):
"""Schema for file upload data"""
original_filename: str
content_type: str
ocr_data: Optional[dict] = None
class PuzzleResponseIn(Schema):
"""Schema for creating a puzzle response"""
puzzle_id: int
puzzle_name: str
cost: Optional[int] = None
cycles: Optional[int] = None
area: Optional[int] = None
needs_manual_validation: bool = False
ocr_confidence_cost: Optional[float] = None
ocr_confidence_cycles: Optional[float] = None
ocr_confidence_area: Optional[float] = None
class SubmissionIn(Schema):
"""Schema for creating a submission"""
notes: Optional[str] = None
manual_validation_requested: bool = False
responses: List[PuzzleResponseIn]
# Output Schemas
class SubmissionFileOut(ModelSchema):
"""Schema for submission file output"""
file_url: Optional[str]
class Meta:
model = SubmissionFile
fields = [
"id",
"original_filename",
"file_size",
"content_type",
"ocr_processed",
"ocr_raw_data",
"ocr_error",
"created_at",
]
class PuzzleResponseOut(ModelSchema):
"""Schema for puzzle response output"""
files: List[SubmissionFileOut]
final_cost: Optional[int]
final_cycles: Optional[int]
final_area: Optional[int]
class Meta:
model = PuzzleResponse
fields = [
"id",
"puzzle",
"puzzle_name",
"cost",
"cycles",
"area",
"needs_manual_validation",
"ocr_confidence_cost",
"ocr_confidence_cycles",
"ocr_confidence_area",
"validated_cost",
"validated_cycles",
"validated_area",
"created_at",
"updated_at",
]
class SubmissionOut(ModelSchema):
"""Schema for submission output"""
responses: List[PuzzleResponseOut]
total_responses: int
needs_validation: bool
class Meta:
model = Submission
fields = [
"id",
"user",
"notes",
"is_validated",
"validated_by",
"validated_at",
"manual_validation_requested",
"created_at",
"updated_at",
]
class SubmissionListOut(Schema):
"""Schema for submission list output"""
id: UUID
# user: int
notes: Optional[str]
total_responses: int
needs_validation: bool
is_validated: bool
created_at: datetime
updated_at: datetime
# Validation Schemas
class ValidationIn(Schema):
"""Schema for manual validation input"""
puzzle: Optional[int] = None
validated_cost: Optional[int] = None
validated_cycles: Optional[int] = None
validated_area: Optional[int] = None
# Collection Schemas
class SteamCollectionItemOut(ModelSchema):
"""Schema for Steam collection item output"""
steam_url: str
class Meta:
model = SteamCollectionItem
fields = [
"id",
"steam_item_id",
"title",
"author_name",
"description",
"tags",
"order_index",
"created_at",
"updated_at",
]
# Error Schemas
class ErrorOut(Schema):
"""Schema for error responses"""
detail: str
code: Optional[str] = None
class ValidationErrorOut(Schema):
"""Schema for validation error responses"""
detail: str
errors: dict
# User Schemas
class UserInfoOut(Schema):
"""Schema for user information output"""
id: Optional[int] = None
username: Optional[str] = None
first_name: Optional[str] = None
last_name: Optional[str] = None
email: Optional[str] = None
is_authenticated: bool
is_staff: bool
is_superuser: bool
cas_groups: Optional[List[str]] = None
polylan_submitter/submissions/tests.py
+1
View File
@@ -0,0 +1 @@
# Create your tests here.
polylan_submitter/submissions/utils.py
+516
View File
@@ -0,0 +1,516 @@
"""
Utilities for fetching Steam Workshop collection data using Steam Web API
"""
import re
import requests
from submissions.models import SteamCollection, SteamCollectionItem, SubmissionFile
from datetime import datetime
from django.utils import timezone
from django.conf import settings
from typing import Dict, List, Optional, Tuple
import logging
from PIL import Image
import cv2
import pytesseract
import os
logger = logging.getLogger(__name__)
class SteamAPIClient:
"""Client for interacting with Steam Web API"""
BASE_URL = "https://api.steampowered.com"
def __init__(self, api_key: Optional[str] = None):
# Priority: parameter > database > settings > environment
self.api_key = (
api_key
or self._get_api_key_from_db()
or getattr(settings, "STEAM_API_KEY", None)
)
self.session = requests.Session()
if not self.api_key:
logger.warning("No Steam API key provided. Some features may be limited.")
def _get_api_key_from_db(self) -> Optional[str]:
"""Get active API key from database"""
try:
from .models import SteamAPIKey
api_key_obj = SteamAPIKey.get_active_key()
if api_key_obj:
# Update last_used timestamp
from django.utils import timezone
api_key_obj.last_used = timezone.now()
api_key_obj.save(update_fields=["last_used"])
return api_key_obj.api_key
except Exception as e:
logger.debug(f"Could not fetch API key from database: {e}")
return None
def get_published_file_details(self, file_ids: List[str]) -> Dict:
"""
Get details for published files (collections/items) using Steam Web API
Args:
file_ids: List of Steam Workshop file IDs
Returns:
API response data
"""
url = f"{self.BASE_URL}/ISteamRemoteStorage/GetPublishedFileDetails/v1/"
# Prepare form data for POST request
data = {
"itemcount": len(file_ids),
}
# Add each file ID
for i, file_id in enumerate(file_ids):
data[f"publishedfileids[{i}]"] = file_id
try:
response = self.session.post(url, data=data, timeout=30)
response.raise_for_status()
return response.json()
except requests.RequestException as e:
logger.error(f"Failed to fetch Steam API data: {e}")
raise
class SteamCollectionFetcher:
"""Utility class for fetching Steam Workshop collection data using Steam API"""
def __init__(self, api_key: Optional[str] = None):
self.api_client = SteamAPIClient(api_key)
def extract_collection_id(self, url: str) -> Optional[str]:
"""
Extract Steam collection ID from various URL formats
Args:
url: Steam Workshop collection URL
Returns:
Collection ID as string, or None if not found
"""
# Handle different URL formats
patterns = [
r"steamcommunity\.com/workshop/filedetails/\?id=(\d+)",
r"steamcommunity\.com/sharedfiles/filedetails/\?id=(\d+)",
r"id=(\d+)",
]
for pattern in patterns:
match = re.search(pattern, url)
if match:
return match.group(1)
return None
def fetch_collection_data(self, url: str) -> Dict:
"""
Fetch collection data from Steam Web API
Args:
url: Steam Workshop collection URL
Returns:
Dictionary containing collection data
Raises:
requests.RequestException: If API request fails
ValueError: If collection ID cannot be extracted or data is invalid
"""
collection_id = self.extract_collection_id(url)
if not collection_id:
raise ValueError(f"Cannot extract collection ID from URL: {url}")
# Fetch collection details from Steam API
api_response = self.api_client.get_published_file_details([collection_id])
if "response" not in api_response:
raise ValueError("Invalid API response format")
response_data = api_response["response"]
if (
"publishedfiledetails" not in response_data
or not response_data["publishedfiledetails"]
):
raise ValueError("No collection data found in API response")
collection_data = response_data["publishedfiledetails"][0]
# Check if collection exists and is accessible
if collection_data.get("result") != 1:
raise ValueError(
f"Collection not found or inaccessible (result: {collection_data.get('result')})"
)
return self._parse_api_collection_data(collection_data, collection_id, url)
def _parse_api_collection_data(
self, api_data: Dict, collection_id: str, url: str
) -> Dict:
"""
Parse collection data from Steam API response
Args:
api_data: Steam API response data for the collection
collection_id: Steam collection ID
url: Original URL
Returns:
Dictionary containing parsed collection data
"""
data = {
"steam_id": collection_id,
"url": url,
"title": api_data.get("title", ""),
"description": api_data.get("description", ""),
"author_name": "",
"author_steam_id": str(api_data.get("creator", "")),
"total_items": 0,
"unique_visitors": api_data.get("views", 0),
"current_favorites": api_data.get("favorited", 0),
"total_favorites": api_data.get("lifetime_favorited", 0),
"steam_created_date": None,
"steam_updated_date": None,
"items": [],
}
# Parse timestamps
if "time_created" in api_data:
data["steam_created_date"] = timezone.make_aware(
datetime.fromtimestamp(api_data["time_created"])
)
if "time_updated" in api_data:
data["steam_updated_date"] = timezone.make_aware(
datetime.fromtimestamp(api_data["time_updated"])
)
# Get author name if we have Steam ID
if data["author_steam_id"]:
try:
author_info = self._get_user_info(data["author_steam_id"])
if author_info:
data["author_name"] = author_info.get("personaname", "")
except Exception as e:
logger.debug(f"Could not fetch author info: {e}")
# Fetch collection items using GetCollectionDetails API
data["items"] = self._fetch_collection_items_via_api(collection_id)
data["total_items"] = len(data["items"])
return data
def _get_user_info(self, steam_id: str) -> Optional[Dict]:
"""
Get user information from Steam API
Args:
steam_id: Steam user ID
Returns:
User info dictionary or None if not available
"""
if not self.api_client.api_key:
return None
url = f"{self.api_client.BASE_URL}/ISteamUser/GetPlayerSummaries/v0002/"
params = {"key": self.api_client.api_key, "steamids": steam_id}
try:
response = self.api_client.session.get(url, params=params, timeout=10)
response.raise_for_status()
data = response.json()
if (
"response" in data
and "players" in data["response"]
and data["response"]["players"]
):
return data["response"]["players"][0]
except Exception as e:
logger.debug(f"Failed to fetch user info for {steam_id}: {e}")
return None
def _fetch_collection_items_via_api(self, collection_id: str) -> List[Dict]:
"""
Fetch collection items using GetCollectionDetails API
Args:
collection_id: Steam collection ID
Returns:
List of item dictionaries
"""
items = []
try:
# Use GetCollectionDetails API to get collection items
url = f"{self.api_client.BASE_URL}/ISteamRemoteStorage/GetCollectionDetails/v1/"
data = {"collectioncount": 1, "publishedfileids[0]": collection_id}
response = self.api_client.session.post(url, data=data, timeout=30)
if response.status_code == 200:
collection_response = response.json()
if (
"response" in collection_response
and "collectiondetails" in collection_response["response"]
):
for collection in collection_response["response"][
"collectiondetails"
]:
if collection.get("result") == 1 and "children" in collection:
# Extract item IDs with their sort order
child_items = []
for child in collection["children"]:
if "publishedfileid" in child:
child_items.append(
{
"id": str(child["publishedfileid"]),
"sort_order": child.get("sortorder", 0),
}
)
# Sort by sort order to maintain collection order
child_items.sort(key=lambda x: x["sort_order"])
item_ids = [item["id"] for item in child_items]
if item_ids:
items = self._fetch_items_by_ids(item_ids)
except Exception as e:
logger.error(f"Failed to fetch collection items via API: {e}")
return items
def _fetch_items_by_ids(self, item_ids: List[str]) -> List[Dict]:
"""Fetch item details by their IDs"""
items = []
# Fetch details for all items in batches (Steam API has limits)
batch_size = 20 # Conservative batch size
for i in range(0, len(item_ids), batch_size):
batch_ids = item_ids[i : i + batch_size]
try:
api_response = self.api_client.get_published_file_details(batch_ids)
if (
"response" in api_response
and "publishedfiledetails" in api_response["response"]
):
for j, item_data in enumerate(
api_response["response"]["publishedfiledetails"]
):
item_id = item_data.get("publishedfileid", "unknown")
result = item_data.get("result", 0)
if result == 1: # Success
item_info = {
"steam_item_id": str(item_id),
"title": item_data.get("title", ""),
"author_name": "",
"author_steam_id": str(item_data.get("creator", "")),
"description": item_data.get("description", ""),
"tags": [
tag.get("tag", "")
for tag in item_data.get("tags", [])
],
"order_index": i + j,
}
# Get author name if available
if item_info["author_steam_id"]:
try:
author_info = self._get_user_info(
item_info["author_steam_id"]
)
if author_info:
item_info["author_name"] = author_info.get(
"personaname", ""
)
except Exception as e:
logger.debug(
f"Could not fetch item author info: {e}"
)
items.append(item_info)
else:
# Log failed items
logger.warning(
f"Failed to fetch item {item_id}: result={result}, ban_reason={item_data.get('ban_reason', 'N/A')}"
)
except Exception as e:
logger.error(f"Failed to fetch batch of collection items: {e}")
continue
return items
def fetch_steam_collection(url: str) -> Dict:
"""
Convenience function to fetch Steam collection data
Args:
url: Steam Workshop collection URL
Returns:
Dictionary containing collection data
"""
fetcher = SteamCollectionFetcher()
return fetcher.fetch_collection_data(url)
def create_or_update_collection(url: str) -> Tuple[SteamCollection, bool]:
"""
Create or update a Steam collection in the database
Args:
url: Steam Workshop collection URL
Returns:
Tuple of (SteamCollection instance, created_flag)
Raises:
ValueError: If collection cannot be fetched or parsed
"""
from .models import SteamCollection, SteamCollectionItem
# Fetch data from Steam
data = fetch_steam_collection(url)
# Create or update collection
collection, created = SteamCollection.objects.update_or_create(
steam_id=data["steam_id"],
defaults={
"url": data["url"],
"title": data["title"],
"description": data["description"],
"author_name": data["author_name"],
"author_steam_id": data["author_steam_id"],
"total_items": data["total_items"],
"unique_visitors": data["unique_visitors"],
"current_favorites": data["current_favorites"],
"total_favorites": data["total_favorites"],
"steam_created_date": data["steam_created_date"],
"steam_updated_date": data["steam_updated_date"],
"last_fetched": timezone.now(),
"fetch_error": "", # Clear any previous errors
},
)
# Update collection items
# First, remove existing items
collection.items.all().delete()
# Add new items
for item_data in data["items"]:
SteamCollectionItem.objects.create(
collection=collection,
steam_item_id=item_data["steam_item_id"],
title=item_data["title"],
author_name=item_data["author_name"],
author_steam_id=item_data["author_steam_id"],
description=item_data["description"],
tags=item_data["tags"],
order_index=item_data["order_index"],
)
return collection, created
def verify_ocr_data_for_file(file: str) -> tuple[str, int, int, int]:
# Convert GIF to JPG
with Image.open(file) as img:
# width, height = img.size
img.seek(0)
rgb_img = img.convert("RGB")
rgb_img.save("temp.jpg", "JPEG")
# Read image from which text needs to be extracted
img = cv2.imread("temp.jpg")
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# Optional: resize for better OCR
gray = cv2.resize(gray, None, fx=1, fy=1, interpolation=cv2.INTER_CUBIC)
# Manually crop regions based on known layout (x, y, w, h)
regions = [
(15, 600, 330, 28), # PUZZLE NAME
(412, 603, 65, 22), # COST
(577, 603, 65, 22), # CYCLES
(739, 603, 65, 22), # AREA
]
output = img.copy()
def find_text(dims, gray, output, content):
x, y, w, h = dims
roi = gray[y : y + h, x : x + w]
roi = cv2.bitwise_not(roi)
if content == "digits" or content == "digits_with_6":
config = "--oem 3 --psm 7 -c tessedit_char_whitelist=0123456789"
else:
config = "--oem 3 --psm 7"
text = pytesseract.image_to_string(roi, config=config).strip()
# Remove the extra 6 (actually the G for Gold) for cost value
if content == "digits_with_6":
text = text[:-1]
cv2.rectangle(output, (x, y), (x + w, y + h), (0, 255, 0), 2)
return text
puzzle = find_text(regions[0], gray, output, "letters")
cost = find_text(regions[1], gray, output, "digits_with_6")
cycles = find_text(regions[2], gray, output, "digits")
area = find_text(regions[3], gray, output, "digits")
# Save image with green rectangles around the considered zones, for debug purposes
# cv2.imwrite("output_debug.jpg", output)
os.remove("temp.jpg")
return puzzle, int(cost), int(cycles), int(area)
def verify_and_validate_ocr_date_for_submission(file: SubmissionFile):
ocr_data = verify_ocr_data_for_file(file.file.path)
r = file.response
print(
f"{r.submission.user}: ({r.cost: >4} {r.cycles: >4} {r.area: >4}) -> ({ocr_data[1]: >4} {ocr_data[2]: >4} {ocr_data[3]: >4})"
)
if puzzle := SteamCollectionItem.objects.filter(title=ocr_data[0]).first():
# print(f"{r.puzzle.title} -> {ocr_data[0]}")
r.puzzle = puzzle
valid_count = 0
for index, field in enumerate(["cost", "cycles", "area"]):
value = getattr(r, field, -1)
# print(f"{value} -> {ocr_data[index + 1]}")
if value == ocr_data[index + 1]:
setattr(r, f"validated_{field}", value)
valid_count += 1
else:
setattr(r, field, ocr_data[index + 1])
r.needs_manual_validation = valid_count != 3
r.save()
polylan_submitter/submissions/views.py
+1
View File
@@ -0,0 +1 @@
# Create your views here.