Legrems/opus-submitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ruff + submissions validation

Browse Source
This commit is contained in:
Legrems
2025-10-30 14:43:19 +01:00
parent 15de496501
commit 0e1e77c2dd
36 changed files with 1025 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files
opus_submitter/submissions/admin.py
+158 -82
View File
@@ -2,8 +2,12 @@ from django.contrib import admin
from django.utils.html import format_html
from django.utils import timezone
from .models import (
SteamAPIKey, SteamCollection, SteamCollectionItem,
Submission, PuzzleResponse, SubmissionFile
SteamAPIKey,
SteamCollection,
SteamCollectionItem,
Submission,
PuzzleResponse,
SubmissionFile,
)
@@ -151,7 +155,14 @@ class SubmissionFileInline(admin.TabularInline):
model = SubmissionFile
extra = 0
readonly_fields = ["file_size", "content_type", "ocr_processed", "created_at"]
fields = ["file", "original_filename", "file_size", "content_type", "ocr_processed", "ocr_error"]
fields = [
"file",
"original_filename",
"file_size",
"content_type",
"ocr_processed",
"ocr_error",
]
class PuzzleResponseInline(admin.TabularInline):
@@ -159,43 +170,73 @@ class PuzzleResponseInline(admin.TabularInline):
extra = 0
readonly_fields = ["created_at", "updated_at"]
fields = [
"puzzle", "puzzle_name", "cost", "cycles", "area",
"needs_manual_validation", "ocr_confidence_cost", "ocr_confidence_cycles", "ocr_confidence_area"
"puzzle",
"puzzle_name",
"cost",
"cycles",
"area",
"needs_manual_validation",
"ocr_confidence_cost",
"ocr_confidence_cycles",
"ocr_confidence_area",
]
@admin.register(Submission)
class SubmissionAdmin(admin.ModelAdmin):
list_display = [
"id", "user", "total_responses", "needs_validation",
"manual_validation_requested", "is_validated", "created_at"
"id",
"user",
"total_responses",
"needs_validation",
"manual_validation_requested",
"is_validated",
"created_at",
]
list_filter = [
"is_validated", "manual_validation_requested", "created_at", "updated_at"
"is_validated",
"manual_validation_requested",
"created_at",
"updated_at",
]
search_fields = ["id", "user__username", "notes"]
readonly_fields = ["id", "created_at", "updated_at", "total_responses", "needs_validation"]
readonly_fields = [
"id",
"created_at",
"updated_at",
"total_responses",
"needs_validation",
]
inlines = [PuzzleResponseInline]
fieldsets = (
("Basic Information", {
"fields": ("id", "user", "notes")
}),
("Validation", {
"fields": ("manual_validation_requested", "is_validated", "validated_by", "validated_at")
}),
("Statistics", {
"fields": ("total_responses", "needs_validation"),
"classes": ("collapse",)
}),
("Timestamps", {
"fields": ("created_at", "updated_at"),
"classes": ("collapse",)
}),
("Basic Information", {"fields": ("id", "user", "notes")}),
(
"Validation",
{
"fields": (
"manual_validation_requested",
"is_validated",
"validated_by",
"validated_at",
)
},
),
(
"Statistics",
{
"fields": ("total_responses", "needs_validation"),
"classes": ("collapse",),
},
),
(
"Timestamps",
{"fields": ("created_at", "updated_at"), "classes": ("collapse",)},
),
)
actions = ["mark_as_validated"]
def mark_as_validated(self, request, queryset):
"""Mark selected submissions as validated"""
updated = 0
@@ -208,59 +249,82 @@ class SubmissionAdmin(admin.ModelAdmin):
# Also mark all responses as not needing validation
submission.responses.update(needs_manual_validation=False)
updated += 1
self.message_user(request, f"{updated} submissions marked as validated.")
mark_as_validated.short_description = "Mark selected submissions as validated"
@admin.register(PuzzleResponse)
class PuzzleResponseAdmin(admin.ModelAdmin):
list_display = [
"puzzle_name", "submission", "puzzle", "cost", "cycles", "area",
"needs_manual_validation", "created_at"
]
list_filter = [
"needs_manual_validation", "puzzle__collection", "created_at"
"puzzle_name",
"submission",
"puzzle",
"cost",
"cycles",
"area",
"needs_manual_validation",
"created_at",
]
list_filter = ["needs_manual_validation", "puzzle__collection", "created_at"]
search_fields = [
"puzzle_name", "submission__id", "puzzle__title",
"cost", "cycles", "area"
"puzzle_name",
"submission__id",
"puzzle__title",
"cost",
"cycles",
"area",
]
readonly_fields = ["created_at", "updated_at"]
inlines = [SubmissionFileInline]
fieldsets = (
("Basic Information", {
"fields": ("submission", "puzzle", "puzzle_name")
}),
("OCR Data", {
"fields": ("cost", "cycles", "area", "ocr_confidence_cost", "ocr_confidence_cycles", "ocr_confidence_area")
}),
("Validation", {
"fields": (
"needs_manual_validation",
"validated_cost", "validated_cycles", "validated_area"
)
}),
("Timestamps", {
"fields": ("created_at", "updated_at"),
"classes": ("collapse",)
}),
("Basic Information", {"fields": ("submission", "puzzle", "puzzle_name")}),
(
"OCR Data",
{
"fields": (
"cost",
"cycles",
"area",
"ocr_confidence_cost",
"ocr_confidence_cycles",
"ocr_confidence_area",
)
},
),
(
"Validation",
{
"fields": (
"needs_manual_validation",
"validated_cost",
"validated_cycles",
"validated_area",
)
},
),
(
"Timestamps",
{"fields": ("created_at", "updated_at"), "classes": ("collapse",)},
),
)
actions = ["mark_for_validation", "clear_validation_flag"]
def mark_for_validation(self, request, queryset):
"""Mark selected responses as needing validation"""
updated = queryset.update(needs_manual_validation=True)
self.message_user(request, f"{updated} responses marked for validation.")
def clear_validation_flag(self, request, queryset):
"""Clear validation flag for selected responses"""
updated = queryset.update(needs_manual_validation=False)
self.message_user(request, f"{updated} responses cleared from validation queue.")
self.message_user(
request, f"{updated} responses cleared from validation queue."
)
mark_for_validation.short_description = "Mark as needing validation"
clear_validation_flag.short_description = "Clear validation flag"
@@ -268,37 +332,49 @@ class PuzzleResponseAdmin(admin.ModelAdmin):
@admin.register(SubmissionFile)
class SubmissionFileAdmin(admin.ModelAdmin):
list_display = [
"original_filename", "response", "file_size_display",
"content_type", "ocr_processed", "created_at"
]
list_filter = [
"content_type", "ocr_processed", "created_at"
"original_filename",
"response",
"file_size_display",
"content_type",
"ocr_processed",
"created_at",
]
list_filter = ["content_type", "ocr_processed", "created_at"]
search_fields = [
"original_filename", "response__puzzle_name",
"response__submission__id"
"original_filename",
"response__puzzle_name",
"response__submission__id",
]
readonly_fields = [
"file_size", "content_type", "ocr_processed",
"created_at", "updated_at", "file_url"
"file_size",
"content_type",
"ocr_processed",
"created_at",
"updated_at",
"file_url",
]
fieldsets = (
("File Information", {
"fields": ("file", "original_filename", "file_size", "content_type", "file_url")
}),
("OCR Processing", {
"fields": ("ocr_processed", "ocr_raw_data", "ocr_error")
}),
("Relationships", {
"fields": ("response",)
}),
("Timestamps", {
"fields": ("created_at", "updated_at"),
"classes": ("collapse",)
}),
(
"File Information",
{
"fields": (
"file",
"original_filename",
"file_size",
"content_type",
"file_url",
)
},
),
("OCR Processing", {"fields": ("ocr_processed", "ocr_raw_data", "ocr_error")}),
("Relationships", {"fields": ("response",)}),
(
"Timestamps",
{"fields": ("created_at", "updated_at"), "classes": ("collapse",)},
),
)
def file_size_display(self, obj):
"""Display file size in human readable format"""
if obj.file_size < 1024:
@@ -307,5 +383,5 @@ class SubmissionFileAdmin(admin.ModelAdmin):
return f"{obj.file_size / 1024:.1f} KB"
else:
return f"{obj.file_size / (1024 * 1024):.1f} MB"
file_size_display.short_description = "File Size"
opus_submitter/submissions/api.py
+16 -6
View File
@@ -69,17 +69,27 @@ def create_submission(
with transaction.atomic():
# Check if any confidence score is below 50% to auto-request validation
auto_request_validation = any(
(response_data.ocr_confidence_cost is not None and response_data.ocr_confidence_cost < 0.5) or
(response_data.ocr_confidence_cycles is not None and response_data.ocr_confidence_cycles < 0.5) or
(response_data.ocr_confidence_area is not None and response_data.ocr_confidence_area < 0.5)
(
response_data.ocr_confidence_cost is not None
and response_data.ocr_confidence_cost < 0.5
)
or (
response_data.ocr_confidence_cycles is not None
and response_data.ocr_confidence_cycles < 0.5
)
or (
response_data.ocr_confidence_area is not None
and response_data.ocr_confidence_area < 0.5
)
for response_data in data.responses
)
# Create the submission
submission = Submission.objects.create(
user=request.user if request.user.is_authenticated else None,
notes=data.notes,
manual_validation_requested=data.manual_validation_requested or auto_request_validation,
manual_validation_requested=data.manual_validation_requested
or auto_request_validation,
)
file_index = 0
@@ -100,7 +110,7 @@ def create_submission(
cost=response_data.cost,
cycles=response_data.cycles,
area=response_data.area,
needs_manual_validation=response_data.needs_manual_validation,
needs_manual_validation=data.manual_validation_requested,
ocr_confidence_cost=response_data.ocr_confidence_cost,
ocr_confidence_cycles=response_data.ocr_confidence_cycles,
ocr_confidence_area=response_data.ocr_confidence_area,
opus_submitter/submissions/apps.py
+2 -2
View File
@@ -2,5 +2,5 @@ from django.apps import AppConfig
class SubmissionsConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'submissions'
default_auto_field = "django.db.models.BigAutoField"
name = "submissions"
opus_submitter/submissions/management/commands/fetch_steam_collection.py
+34 -29
View File
@@ -8,40 +8,39 @@ from submissions.models import SteamCollection
class Command(BaseCommand):
help = 'Fetch Steam Workshop collection data and save to database'
help = "Fetch Steam Workshop collection data and save to database"
def add_arguments(self, parser):
parser.add_argument("url", type=str, help="Steam Workshop collection URL")
parser.add_argument(
'url',
"--api-key",
type=str,
help='Steam Workshop collection URL'
help="Steam API key (optional, can also be set via STEAM_API_KEY environment variable)",
)
parser.add_argument(
'--api-key',
type=str,
help='Steam API key (optional, can also be set via STEAM_API_KEY environment variable)'
"--force",
action="store_true",
help="Force refetch even if collection already exists",
)
parser.add_argument(
'--force',
action='store_true',
help='Force refetch even if collection already exists'
)
def handle(self, *args, **options):
url = options['url']
api_key = options.get('api_key')
force = options['force']
url = options["url"]
api_key = options.get("api_key")
force = options["force"]
self.stdout.write(f"Fetching Steam collection from: {url}")
try:
# Check if collection already exists
from submissions.utils import SteamCollectionFetcher
fetcher = SteamCollectionFetcher(api_key)
collection_id = fetcher.extract_collection_id(url)
if collection_id and not force:
existing = SteamCollection.objects.filter(steam_id=collection_id).first()
existing = SteamCollection.objects.filter(
steam_id=collection_id
).first()
if existing:
self.stdout.write(
self.style.WARNING(
@@ -50,10 +49,10 @@ class Command(BaseCommand):
)
)
return
# Fetch and create/update collection
collection, created = create_or_update_collection(url)
if created:
self.stdout.write(
self.style.SUCCESS(
@@ -66,27 +65,33 @@ class Command(BaseCommand):
f"Successfully updated collection: {collection.title} (ID: {collection.id})"
)
)
# Display collection info
self.stdout.write("\nCollection Details:")
self.stdout.write(f" Steam ID: {collection.steam_id}")
self.stdout.write(f" Title: {collection.title}")
self.stdout.write(f" Author: {collection.author_name or 'Unknown'}")
self.stdout.write(f" Description: {collection.description[:100]}{'...' if len(collection.description) > 100 else ''}")
self.stdout.write(
f" Description: {collection.description[:100]}{'...' if len(collection.description) > 100 else ''}"
)
self.stdout.write(f" Total Items: {collection.total_items}")
self.stdout.write(f" Unique Visitors: {collection.unique_visitors}")
self.stdout.write(f" Current Favorites: {collection.current_favorites}")
self.stdout.write(f" Total Favorites: {collection.total_favorites}")
if collection.items.exists():
self.stdout.write(f"\nCollection Items ({collection.items.count()}):")
for item in collection.items.all()[:10]: # Show first 10 items
self.stdout.write(f" - {item.title} (Steam ID: {item.steam_item_id})")
self.stdout.write(
f" - {item.title} (Steam ID: {item.steam_item_id})"
)
if collection.items.count() > 10:
self.stdout.write(f" ... and {collection.items.count() - 10} more items")
self.stdout.write(
f" ... and {collection.items.count() - 10} more items"
)
else:
self.stdout.write("\nNo items found in collection.")
except Exception as e:
raise CommandError(f"Failed to fetch collection: {e}")
opus_submitter/submissions/migrations/0001_initial.py
+191 -44
View File
@@ -5,68 +5,215 @@ from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
]
dependencies = []
operations = [
migrations.CreateModel(
name='Collection',
name="Collection",
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('url', models.URLField()),
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("url", models.URLField()),
],
),
migrations.CreateModel(
name='SteamCollection',
name="SteamCollection",
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('steam_id', models.CharField(help_text='Steam collection ID from URL', max_length=50, unique=True)),
('url', models.URLField(help_text='Full Steam Workshop collection URL')),
('title', models.CharField(blank=True, help_text='Collection title', max_length=255)),
('description', models.TextField(blank=True, help_text='Collection description')),
('author_name', models.CharField(blank=True, help_text='Steam username of collection creator', max_length=100)),
('author_steam_id', models.CharField(blank=True, help_text='Steam ID of collection creator', max_length=50)),
('total_items', models.PositiveIntegerField(default=0, help_text='Number of items in collection')),
('unique_visitors', models.PositiveIntegerField(default=0, help_text='Number of unique visitors')),
('current_favorites', models.PositiveIntegerField(default=0, help_text='Current number of favorites')),
('total_favorites', models.PositiveIntegerField(default=0, help_text='Total unique favorites')),
('steam_created_date', models.DateTimeField(blank=True, help_text='When collection was created on Steam', null=True)),
('steam_updated_date', models.DateTimeField(blank=True, help_text='When collection was last updated on Steam', null=True)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('last_fetched', models.DateTimeField(blank=True, help_text='When data was last fetched from Steam', null=True)),
('is_active', models.BooleanField(default=True, help_text='Whether this collection is actively tracked')),
('fetch_error', models.TextField(blank=True, help_text='Last error encountered when fetching data')),
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"steam_id",
models.CharField(
help_text="Steam collection ID from URL",
max_length=50,
unique=True,
),
),
(
"url",
models.URLField(help_text="Full Steam Workshop collection URL"),
),
(
"title",
models.CharField(
blank=True, help_text="Collection title", max_length=255
),
),
(
"description",
models.TextField(blank=True, help_text="Collection description"),
),
(
"author_name",
models.CharField(
blank=True,
help_text="Steam username of collection creator",
max_length=100,
),
),
(
"author_steam_id",
models.CharField(
blank=True,
help_text="Steam ID of collection creator",
max_length=50,
),
),
(
"total_items",
models.PositiveIntegerField(
default=0, help_text="Number of items in collection"
),
),
(
"unique_visitors",
models.PositiveIntegerField(
default=0, help_text="Number of unique visitors"
),
),
(
"current_favorites",
models.PositiveIntegerField(
default=0, help_text="Current number of favorites"
),
),
(
"total_favorites",
models.PositiveIntegerField(
default=0, help_text="Total unique favorites"
),
),
(
"steam_created_date",
models.DateTimeField(
blank=True,
help_text="When collection was created on Steam",
null=True,
),
),
(
"steam_updated_date",
models.DateTimeField(
blank=True,
help_text="When collection was last updated on Steam",
null=True,
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"last_fetched",
models.DateTimeField(
blank=True,
help_text="When data was last fetched from Steam",
null=True,
),
),
(
"is_active",
models.BooleanField(
default=True,
help_text="Whether this collection is actively tracked",
),
),
(
"fetch_error",
models.TextField(
blank=True,
help_text="Last error encountered when fetching data",
),
),
],
options={
'verbose_name': 'Steam Collection',
'verbose_name_plural': 'Steam Collections',
'ordering': ['-created_at'],
"verbose_name": "Steam Collection",
"verbose_name_plural": "Steam Collections",
"ordering": ["-created_at"],
},
),
migrations.CreateModel(
name='SteamCollectionItem',
name="SteamCollectionItem",
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('steam_item_id', models.CharField(help_text='Steam Workshop item ID', max_length=50)),
('title', models.CharField(blank=True, help_text='Item title', max_length=255)),
('author_name', models.CharField(blank=True, help_text='Steam username of item creator', max_length=100)),
('author_steam_id', models.CharField(blank=True, help_text='Steam ID of item creator', max_length=50)),
('description', models.TextField(blank=True, help_text='Item description')),
('tags', models.JSONField(blank=True, default=list, help_text='Item tags as JSON array')),
('order_index', models.PositiveIntegerField(default=0, help_text='Order of item in collection')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='submissions.steamcollection')),
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"steam_item_id",
models.CharField(help_text="Steam Workshop item ID", max_length=50),
),
(
"title",
models.CharField(
blank=True, help_text="Item title", max_length=255
),
),
(
"author_name",
models.CharField(
blank=True,
help_text="Steam username of item creator",
max_length=100,
),
),
(
"author_steam_id",
models.CharField(
blank=True, help_text="Steam ID of item creator", max_length=50
),
),
(
"description",
models.TextField(blank=True, help_text="Item description"),
),
(
"tags",
models.JSONField(
blank=True, default=list, help_text="Item tags as JSON array"
),
),
(
"order_index",
models.PositiveIntegerField(
default=0, help_text="Order of item in collection"
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"collection",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="items",
to="submissions.steamcollection",
),
),
],
options={
'verbose_name': 'Steam Collection Item',
'verbose_name_plural': 'Steam Collection Items',
'ordering': ['collection', 'order_index'],
'unique_together': {('collection', 'steam_item_id')},
"verbose_name": "Steam Collection Item",
"verbose_name_plural": "Steam Collection Items",
"ordering": ["collection", "order_index"],
"unique_together": {("collection", "steam_item_id")},
},
),
]
opus_submitter/submissions/migrations/0002_delete_collection.py
+2 -3
View File
@@ -4,13 +4,12 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('submissions', '0001_initial'),
("submissions", "0001_initial"),
]
operations = [
migrations.DeleteModel(
name='Collection',
name="Collection",
),
]
opus_submitter/submissions/migrations/0003_steamapikey.py
+52 -14
View File
@@ -4,28 +4,66 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('submissions', '0002_delete_collection'),
("submissions", "0002_delete_collection"),
]
operations = [
migrations.CreateModel(
name='SteamAPIKey',
name="SteamAPIKey",
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(help_text="Descriptive name for this API key (e.g., 'Production Key', 'Development Key')", max_length=100, unique=True)),
('api_key', models.CharField(help_text='Steam Web API key from https://steamcommunity.com/dev/apikey', max_length=64)),
('is_active', models.BooleanField(default=True, help_text='Whether this API key should be used')),
('description', models.TextField(blank=True, help_text='Optional description or notes about this API key')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('last_used', models.DateTimeField(blank=True, help_text='When this API key was last used', null=True)),
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"name",
models.CharField(
help_text="Descriptive name for this API key (e.g., 'Production Key', 'Development Key')",
max_length=100,
unique=True,
),
),
(
"api_key",
models.CharField(
help_text="Steam Web API key from https://steamcommunity.com/dev/apikey",
max_length=64,
),
),
(
"is_active",
models.BooleanField(
default=True, help_text="Whether this API key should be used"
),
),
(
"description",
models.TextField(
blank=True,
help_text="Optional description or notes about this API key",
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"last_used",
models.DateTimeField(
blank=True,
help_text="When this API key was last used",
null=True,
),
),
],
options={
'verbose_name': 'Steam API Key',
'verbose_name_plural': 'Steam API Keys',
'ordering': ['-is_active', 'name'],
"verbose_name": "Steam API Key",
"verbose_name_plural": "Steam API Keys",
"ordering": ["-is_active", "name"],
},
),
]
opus_submitter/submissions/migrations/0004_submission_puzzleresponse_submissionfile.py
+218 -48
View File
@@ -8,75 +8,245 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('submissions', '0003_steamapikey'),
("submissions", "0003_steamapikey"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='Submission',
name="Submission",
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('notes', models.TextField(blank=True, help_text='Optional notes about the submission')),
('is_validated', models.BooleanField(default=False, help_text='Whether this submission has been manually validated')),
('validated_at', models.DateTimeField(blank=True, help_text='When this submission was validated', null=True)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('user', models.ForeignKey(blank=True, help_text='User who made the submission (null for anonymous)', null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
('validated_by', models.ForeignKey(blank=True, help_text='Admin user who validated this submission', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='validated_submissions', to=settings.AUTH_USER_MODEL)),
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
(
"notes",
models.TextField(
blank=True, help_text="Optional notes about the submission"
),
),
(
"is_validated",
models.BooleanField(
default=False,
help_text="Whether this submission has been manually validated",
),
),
(
"validated_at",
models.DateTimeField(
blank=True,
help_text="When this submission was validated",
null=True,
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"user",
models.ForeignKey(
blank=True,
help_text="User who made the submission (null for anonymous)",
null=True,
on_delete=django.db.models.deletion.CASCADE,
to=settings.AUTH_USER_MODEL,
),
),
(
"validated_by",
models.ForeignKey(
blank=True,
help_text="Admin user who validated this submission",
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="validated_submissions",
to=settings.AUTH_USER_MODEL,
),
),
],
options={
'verbose_name': 'Submission',
'verbose_name_plural': 'Submissions',
'ordering': ['-created_at'],
"verbose_name": "Submission",
"verbose_name_plural": "Submissions",
"ordering": ["-created_at"],
},
),
migrations.CreateModel(
name='PuzzleResponse',
name="PuzzleResponse",
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('puzzle_name', models.CharField(help_text='Puzzle name as detected by OCR', max_length=255)),
('cost', models.CharField(blank=True, help_text='Cost value from OCR', max_length=20)),
('cycles', models.CharField(blank=True, help_text='Cycles value from OCR', max_length=20)),
('area', models.CharField(blank=True, help_text='Area value from OCR', max_length=20)),
('needs_manual_validation', models.BooleanField(default=False, help_text='Whether OCR failed and manual validation is needed')),
('ocr_confidence_score', models.FloatField(blank=True, help_text='OCR confidence score (0.0 to 1.0)', null=True)),
('validated_cost', models.CharField(blank=True, help_text='Manually validated cost value', max_length=20)),
('validated_cycles', models.CharField(blank=True, help_text='Manually validated cycles value', max_length=20)),
('validated_area', models.CharField(blank=True, help_text='Manually validated area value', max_length=20)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('puzzle', models.ForeignKey(help_text='The puzzle this response is for', on_delete=django.db.models.deletion.CASCADE, related_name='responses', to='submissions.steamcollectionitem')),
('submission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='responses', to='submissions.submission')),
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"puzzle_name",
models.CharField(
help_text="Puzzle name as detected by OCR", max_length=255
),
),
(
"cost",
models.CharField(
blank=True, help_text="Cost value from OCR", max_length=20
),
),
(
"cycles",
models.CharField(
blank=True, help_text="Cycles value from OCR", max_length=20
),
),
(
"area",
models.CharField(
blank=True, help_text="Area value from OCR", max_length=20
),
),
(
"needs_manual_validation",
models.BooleanField(
default=False,
help_text="Whether OCR failed and manual validation is needed",
),
),
(
"ocr_confidence_score",
models.FloatField(
blank=True,
help_text="OCR confidence score (0.0 to 1.0)",
null=True,
),
),
(
"validated_cost",
models.CharField(
blank=True,
help_text="Manually validated cost value",
max_length=20,
),
),
(
"validated_cycles",
models.CharField(
blank=True,
help_text="Manually validated cycles value",
max_length=20,
),
),
(
"validated_area",
models.CharField(
blank=True,
help_text="Manually validated area value",
max_length=20,
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"puzzle",
models.ForeignKey(
help_text="The puzzle this response is for",
on_delete=django.db.models.deletion.CASCADE,
related_name="responses",
to="submissions.steamcollectionitem",
),
),
(
"submission",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="responses",
to="submissions.submission",
),
),
],
options={
'verbose_name': 'Puzzle Response',
'verbose_name_plural': 'Puzzle Responses',
'ordering': ['submission', 'puzzle__order_index'],
'unique_together': {('submission', 'puzzle')},
"verbose_name": "Puzzle Response",
"verbose_name_plural": "Puzzle Responses",
"ordering": ["submission", "puzzle__order_index"],
"unique_together": {("submission", "puzzle")},
},
),
migrations.CreateModel(
name='SubmissionFile',
name="SubmissionFile",
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('file', models.FileField(help_text='Uploaded file (image/gif)', upload_to=submissions.models.submission_file_upload_path)),
('original_filename', models.CharField(help_text='Original filename as uploaded by user', max_length=255)),
('file_size', models.PositiveIntegerField(help_text='File size in bytes')),
('content_type', models.CharField(help_text='MIME type of the file', max_length=100)),
('ocr_processed', models.BooleanField(default=False, help_text='Whether OCR has been processed for this file')),
('ocr_raw_data', models.JSONField(blank=True, help_text='Raw OCR data as JSON', null=True)),
('ocr_error', models.TextField(blank=True, help_text='OCR processing error message')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('response', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='submissions.puzzleresponse')),
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"file",
models.FileField(
help_text="Uploaded file (image/gif)",
upload_to=submissions.models.submission_file_upload_path,
),
),
(
"original_filename",
models.CharField(
help_text="Original filename as uploaded by user",
max_length=255,
),
),
(
"file_size",
models.PositiveIntegerField(help_text="File size in bytes"),
),
(
"content_type",
models.CharField(help_text="MIME type of the file", max_length=100),
),
(
"ocr_processed",
models.BooleanField(
default=False,
help_text="Whether OCR has been processed for this file",
),
),
(
"ocr_raw_data",
models.JSONField(
blank=True, help_text="Raw OCR data as JSON", null=True
),
),
(
"ocr_error",
models.TextField(
blank=True, help_text="OCR processing error message"
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"response",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="files",
to="submissions.puzzleresponse",
),
),
],
options={
'verbose_name': 'Submission File',
'verbose_name_plural': 'Submission Files',
'ordering': ['response', 'created_at'],
"verbose_name": "Submission File",
"verbose_name_plural": "Submission Files",
"ordering": ["response", "created_at"],
},
),
]
opus_submitter/submissions/migrations/0005_alter_submission_notes.py
+6 -5
View File
@@ -4,15 +4,16 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('submissions', '0004_submission_puzzleresponse_submissionfile'),
("submissions", "0004_submission_puzzleresponse_submissionfile"),
]
operations = [
migrations.AlterField(
model_name='submission',
name='notes',
field=models.TextField(blank=True, help_text='Optional notes about the submission', null=True),
model_name="submission",
name="notes",
field=models.TextField(
blank=True, help_text="Optional notes about the submission", null=True
),
),
]
opus_submitter/submissions/migrations/0006_remove_puzzleresponse_ocr_confidence_score_and_more.py
+24 -13
View File
@@ -4,29 +4,40 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('submissions', '0005_alter_submission_notes'),
("submissions", "0005_alter_submission_notes"),
]
operations = [
migrations.RemoveField(
model_name='puzzleresponse',
name='ocr_confidence_score',
model_name="puzzleresponse",
name="ocr_confidence_score",
),
migrations.AddField(
model_name='puzzleresponse',
name='ocr_confidence_area',
field=models.FloatField(blank=True, help_text='OCR confidence score for area (0.0 to 1.0)', null=True),
model_name="puzzleresponse",
name="ocr_confidence_area",
field=models.FloatField(
blank=True,
help_text="OCR confidence score for area (0.0 to 1.0)",
null=True,
),
),
migrations.AddField(
model_name='puzzleresponse',
name='ocr_confidence_cost',
field=models.FloatField(blank=True, help_text='OCR confidence score for cost (0.0 to 1.0)', null=True),
model_name="puzzleresponse",
name="ocr_confidence_cost",
field=models.FloatField(
blank=True,
help_text="OCR confidence score for cost (0.0 to 1.0)",
null=True,
),
),
migrations.AddField(
model_name='puzzleresponse',
name='ocr_confidence_cycles',
field=models.FloatField(blank=True, help_text='OCR confidence score for cycles (0.0 to 1.0)', null=True),
model_name="puzzleresponse",
name="ocr_confidence_cycles",
field=models.FloatField(
blank=True,
help_text="OCR confidence score for cycles (0.0 to 1.0)",
null=True,
),
),
]
opus_submitter/submissions/migrations/0007_submission_manual_validation_requested.py
+7 -5
View File
@@ -4,15 +4,17 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('submissions', '0006_remove_puzzleresponse_ocr_confidence_score_and_more'),
("submissions", "0006_remove_puzzleresponse_ocr_confidence_score_and_more"),
]
operations = [
migrations.AddField(
model_name='submission',
name='manual_validation_requested',
field=models.BooleanField(default=False, help_text='Whether the user specifically requested manual validation'),
model_name="submission",
name="manual_validation_requested",
field=models.BooleanField(
default=False,
help_text="Whether the user specifically requested manual validation",
),
),
]
opus_submitter/submissions/models.py
+3 -5
View File
@@ -1,9 +1,7 @@
from django.db import models
from django.contrib.auth import get_user_model
from django.utils import timezone
from django.core.exceptions import ValidationError
import uuid
import os
User = get_user_model()
@@ -242,11 +240,11 @@ class Submission(models.Model):
validated_at = models.DateTimeField(
null=True, blank=True, help_text="When this submission was validated"
)
# Manual validation request
manual_validation_requested = models.BooleanField(
default=False,
help_text="Whether the user specifically requested manual validation"
default=False,
help_text="Whether the user specifically requested manual validation",
)
# Timestamps
opus_submitter/submissions/schemas.py
+1 -2
View File
@@ -1,5 +1,4 @@
from ninja import Schema, ModelSchema, File
from ninja.files import UploadedFile
from ninja import Schema, ModelSchema
from typing import List, Optional
from datetime import datetime
from uuid import UUID
opus_submitter/submissions/tests.py
-2
View File
@@ -1,3 +1 @@
from django.test import TestCase
# Create your tests here.
opus_submitter/submissions/utils.py
+45 -33
View File
@@ -4,6 +4,7 @@ Utilities for fetching Steam Workshop collection data using Steam Web API
import re
import requests
from submissions.models import SteamCollection
from datetime import datetime
from django.utils import timezone
from django.conf import settings
@@ -20,22 +21,28 @@ class SteamAPIClient:
def __init__(self, api_key: Optional[str] = None):
# Priority: parameter > database > settings > environment
self.api_key = api_key or self._get_api_key_from_db() or getattr(settings, "STEAM_API_KEY", None)
self.api_key = (
api_key
or self._get_api_key_from_db()
or getattr(settings, "STEAM_API_KEY", None)
)
self.session = requests.Session()
if not self.api_key:
logger.warning("No Steam API key provided. Some features may be limited.")
def _get_api_key_from_db(self) -> Optional[str]:
"""Get active API key from database"""
try:
from .models import SteamAPIKey
api_key_obj = SteamAPIKey.get_active_key()
if api_key_obj:
# Update last_used timestamp
from django.utils import timezone
api_key_obj.last_used = timezone.now()
api_key_obj.save(update_fields=['last_used'])
api_key_obj.save(update_fields=["last_used"])
return api_key_obj.api_key
except Exception as e:
logger.debug(f"Could not fetch API key from database: {e}")
@@ -234,55 +241,59 @@ class SteamCollectionFetcher:
def _fetch_collection_items_via_api(self, collection_id: str) -> List[Dict]:
"""
Fetch collection items using GetCollectionDetails API
Args:
collection_id: Steam collection ID
Returns:
List of item dictionaries
"""
items = []
try:
# Use GetCollectionDetails API to get collection items
url = f"{self.api_client.BASE_URL}/ISteamRemoteStorage/GetCollectionDetails/v1/"
data = {
'collectioncount': 1,
'publishedfileids[0]': collection_id
}
data = {"collectioncount": 1, "publishedfileids[0]": collection_id}
response = self.api_client.session.post(url, data=data, timeout=30)
if response.status_code == 200:
collection_response = response.json()
if 'response' in collection_response and 'collectiondetails' in collection_response['response']:
for collection in collection_response['response']['collectiondetails']:
if collection.get('result') == 1 and 'children' in collection:
if (
"response" in collection_response
and "collectiondetails" in collection_response["response"]
):
for collection in collection_response["response"][
"collectiondetails"
]:
if collection.get("result") == 1 and "children" in collection:
# Extract item IDs with their sort order
child_items = []
for child in collection['children']:
if 'publishedfileid' in child:
child_items.append({
'id': str(child['publishedfileid']),
'sort_order': child.get('sortorder', 0)
})
for child in collection["children"]:
if "publishedfileid" in child:
child_items.append(
{
"id": str(child["publishedfileid"]),
"sort_order": child.get("sortorder", 0),
}
)
# Sort by sort order to maintain collection order
child_items.sort(key=lambda x: x['sort_order'])
item_ids = [item['id'] for item in child_items]
child_items.sort(key=lambda x: x["sort_order"])
item_ids = [item["id"] for item in child_items]
if item_ids:
items = self._fetch_items_by_ids(item_ids)
except Exception as e:
logger.error(f"Failed to fetch collection items via API: {e}")
return items
def _fetch_items_by_ids(self, item_ids: List[str]) -> List[Dict]:
"""Fetch item details by their IDs"""
items = []
# Fetch details for all items in batches (Steam API has limits)
batch_size = 20 # Conservative batch size
for i in range(0, len(item_ids), batch_size):
@@ -300,7 +311,7 @@ class SteamCollectionFetcher:
):
item_id = item_data.get("publishedfileid", "unknown")
result = item_data.get("result", 0)
if result == 1: # Success
item_info = {
"steam_item_id": str(item_id),
@@ -333,14 +344,15 @@ class SteamCollectionFetcher:
items.append(item_info)
else:
# Log failed items
logger.warning(f"Failed to fetch item {item_id}: result={result}, ban_reason={item_data.get('ban_reason', 'N/A')}")
logger.warning(
f"Failed to fetch item {item_id}: result={result}, ban_reason={item_data.get('ban_reason', 'N/A')}"
)
except Exception as e:
logger.error(f"Failed to fetch batch of collection items: {e}")
continue
return items
def fetch_steam_collection(url: str) -> Dict:
@@ -357,7 +369,7 @@ def fetch_steam_collection(url: str) -> Dict:
return fetcher.fetch_collection_data(url)
def create_or_update_collection(url: str) -> Tuple["SteamCollection", bool]:
def create_or_update_collection(url: str) -> Tuple[SteamCollection, bool]:
"""
Create or update a Steam collection in the database
opus_submitter/submissions/views.py
-2
View File
@@ -1,3 +1 @@
from django.shortcuts import render
# Create your views here.