Legrems/opus-submitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ruff + submissions validation

Browse Source
This commit is contained in:
Legrems
2025-10-30 14:43:19 +01:00
parent 15de496501
commit 0e1e77c2dd
36 changed files with 1025 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files
opus_submitter/accounts/admin.py
+18 -14
View File
@@ -6,27 +6,31 @@ from .models import CustomUser
@admin.register(CustomUser) @admin.register(CustomUser)
class CustomUserAdmin(UserAdmin): class CustomUserAdmin(UserAdmin):
"""Admin interface for CustomUser.""" """Admin interface for CustomUser."""
# Add custom fields to the user admin # Add custom fields to the user admin
fieldsets = UserAdmin.fieldsets + ( fieldsets = UserAdmin.fieldsets + (
('CAS Information', { (
'fields': ('cas_user_id', 'cas_groups', 'cas_attributes'), "CAS Information",
}), {
"fields": ("cas_user_id", "cas_groups", "cas_attributes"),
},
),
) )
# Add custom fields to the list display # Add custom fields to the list display
list_display = UserAdmin.list_display + ('cas_user_id', 'get_cas_groups_display') list_display = UserAdmin.list_display + ("cas_user_id", "get_cas_groups_display")
# Add search fields # Add search fields
search_fields = UserAdmin.search_fields + ('cas_user_id',) search_fields = UserAdmin.search_fields + ("cas_user_id",)
# Add filters # Add filters
list_filter = UserAdmin.list_filter + ('cas_groups',) list_filter = UserAdmin.list_filter + ("cas_groups",)
# Make CAS fields readonly in admin # Make CAS fields readonly in admin
readonly_fields = ('cas_user_id', 'cas_groups', 'cas_attributes') readonly_fields = ("cas_user_id", "cas_groups", "cas_attributes")
def get_cas_groups_display(self, obj): def get_cas_groups_display(self, obj):
"""Display CAS groups in admin list.""" """Display CAS groups in admin list."""
return obj.get_cas_groups_display() return obj.get_cas_groups_display()
get_cas_groups_display.short_description = 'CAS Groups'
get_cas_groups_display.short_description = "CAS Groups"
opus_submitter/accounts/apps.py
+2 -2
View File
@@ -2,5 +2,5 @@ from django.apps import AppConfig
class AccountsConfig(AppConfig): class AccountsConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField' default_auto_field = "django.db.models.BigAutoField"
name = 'accounts' name = "accounts"
opus_submitter/accounts/migrations/0001_initial.py
+113 -23
View File
@@ -7,41 +7,131 @@ from django.db import migrations, models
class Migration(migrations.Migration): class Migration(migrations.Migration):
initial = True initial = True
dependencies = [ dependencies = [
('auth', '0012_alter_user_first_name_max_length'), ("auth", "0012_alter_user_first_name_max_length"),
] ]
operations = [ operations = [
migrations.CreateModel( migrations.CreateModel(
name='CustomUser', name="CustomUser",
fields=[ fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), (
('password', models.CharField(max_length=128, verbose_name='password')), "id",
('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), models.BigAutoField(
('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), auto_created=True,
('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')), primary_key=True,
('first_name', models.CharField(blank=True, max_length=150, verbose_name='first name')), serialize=False,
('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')), verbose_name="ID",
('email', models.EmailField(blank=True, max_length=254, verbose_name='email address')), ),
('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), ),
('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), ("password", models.CharField(max_length=128, verbose_name="password")),
('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), (
('cas_user_id', models.CharField(blank=True, max_length=50, null=True, unique=True)), "last_login",
('cas_groups', models.JSONField(blank=True, default=list)), models.DateTimeField(
('cas_attributes', models.JSONField(blank=True, default=dict)), blank=True, null=True, verbose_name="last login"
('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups')), ),
('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions')), ),
(
"is_superuser",
models.BooleanField(
default=False,
help_text="Designates that this user has all permissions without explicitly assigning them.",
verbose_name="superuser status",
),
),
(
"username",
models.CharField(
error_messages={
"unique": "A user with that username already exists."
},
help_text="Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.",
max_length=150,
unique=True,
validators=[
django.contrib.auth.validators.UnicodeUsernameValidator()
],
verbose_name="username",
),
),
(
"first_name",
models.CharField(
blank=True, max_length=150, verbose_name="first name"
),
),
(
"last_name",
models.CharField(
blank=True, max_length=150, verbose_name="last name"
),
),
(
"email",
models.EmailField(
blank=True, max_length=254, verbose_name="email address"
),
),
(
"is_staff",
models.BooleanField(
default=False,
help_text="Designates whether the user can log into this admin site.",
verbose_name="staff status",
),
),
(
"is_active",
models.BooleanField(
default=True,
help_text="Designates whether this user should be treated as active. Unselect this instead of deleting accounts.",
verbose_name="active",
),
),
(
"date_joined",
models.DateTimeField(
default=django.utils.timezone.now, verbose_name="date joined"
),
),
(
"cas_user_id",
models.CharField(blank=True, max_length=50, null=True, unique=True),
),
("cas_groups", models.JSONField(blank=True, default=list)),
("cas_attributes", models.JSONField(blank=True, default=dict)),
(
"groups",
models.ManyToManyField(
blank=True,
help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.",
related_name="user_set",
related_query_name="user",
to="auth.group",
verbose_name="groups",
),
),
(
"user_permissions",
models.ManyToManyField(
blank=True,
help_text="Specific permissions for this user.",
related_name="user_set",
related_query_name="user",
to="auth.permission",
verbose_name="user permissions",
),
),
], ],
options={ options={
'verbose_name': 'user', "verbose_name": "user",
'verbose_name_plural': 'users', "verbose_name_plural": "users",
'abstract': False, "abstract": False,
}, },
managers=[ managers=[
('objects', django.contrib.auth.models.UserManager()), ("objects", django.contrib.auth.models.UserManager()),
], ],
), ),
] ]
opus_submitter/accounts/models.py
-2
View File
@@ -1,6 +1,5 @@
from django.contrib.auth.models import AbstractUser from django.contrib.auth.models import AbstractUser
from django.db import models from django.db import models
import json
class CustomUser(AbstractUser): class CustomUser(AbstractUser):
@@ -57,4 +56,3 @@ class CustomUser(AbstractUser):
self.cas_attributes = attributes self.cas_attributes = attributes
self.save() self.save()
opus_submitter/accounts/tests.py
-2
View File
@@ -1,3 +1 @@
from django.test import TestCase
# Create your tests here. # Create your tests here.
opus_submitter/accounts/views.py
-2
View File
@@ -1,3 +1 @@
from django.shortcuts import render
# Create your views here. # Create your views here.
opus_submitter/manage.py
+3 -2
View File
@@ -1,12 +1,13 @@
#!/usr/bin/env python #!/usr/bin/env python
"""Django's command-line utility for administrative tasks.""" """Django's command-line utility for administrative tasks."""
import os import os
import sys import sys
def main(): def main():
"""Run administrative tasks.""" """Run administrative tasks."""
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'opus_submitter.settings') os.environ.setdefault("DJANGO_SETTINGS_MODULE", "opus_submitter.settings")
try: try:
from django.core.management import execute_from_command_line from django.core.management import execute_from_command_line
except ImportError as exc: except ImportError as exc:
@@ -18,5 +19,5 @@ def main():
execute_from_command_line(sys.argv) execute_from_command_line(sys.argv)
if __name__ == '__main__': if __name__ == "__main__":
main() main()
opus_submitter/opus_submitter/api.py
+2 -3
View File
@@ -1,5 +1,4 @@
from ninja import NinjaAPI from ninja import NinjaAPI
from ninja.security import django_auth
from submissions.api import router as submissions_router from submissions.api import router as submissions_router
from submissions.schemas import UserInfoOut from submissions.schemas import UserInfoOut
@@ -47,7 +46,7 @@ def api_info(request):
def get_user_info(request): def get_user_info(request):
"""Get current user information""" """Get current user information"""
user = request.user user = request.user
if user.is_authenticated: if user.is_authenticated:
return { return {
"id": user.id, "id": user.id,
@@ -58,7 +57,7 @@ def get_user_info(request):
"is_authenticated": True, "is_authenticated": True,
"is_staff": user.is_staff, "is_staff": user.is_staff,
"is_superuser": user.is_superuser, "is_superuser": user.is_superuser,
"cas_groups": getattr(user, 'cas_groups', []) "cas_groups": getattr(user, "cas_groups", []),
} }
else: else:
return { return {
opus_submitter/opus_submitter/asgi.py
+1 -1
View File
@@ -11,6 +11,6 @@ import os
from django.core.asgi import get_asgi_application from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'opus_submitter.settings') os.environ.setdefault("DJANGO_SETTINGS_MODULE", "opus_submitter.settings")
application = get_asgi_application() application = get_asgi_application()
opus_submitter/opus_submitter/settings.py
+1 -3
View File
@@ -176,6 +176,4 @@ STATICFILES_DIRS = [
os.path.join(BASE_DIR, "static_source/vite"), os.path.join(BASE_DIR, "static_source/vite"),
] ]
JWT_SECRET_KEY = "rooCaimosaicae3Oos2quezieb9rohsem1eufieJoo" from opus_submitter.settingsLocal import * # noqa
from opus_submitter.settingsLocal import *
opus_submitter/opus_submitter/wsgi.py
+1 -1
View File
@@ -11,6 +11,6 @@ import os
from django.core.wsgi import get_wsgi_application from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'opus_submitter.settings') os.environ.setdefault("DJANGO_SETTINGS_MODULE", "opus_submitter.settings")
application = get_wsgi_application() application = get_wsgi_application()
opus_submitter/simple_cas_views.py
+14 -10
View File
@@ -12,35 +12,39 @@ import urllib.parse
class SimpleCASLoginView(View): class SimpleCASLoginView(View):
"""Simple CAS login view.""" """Simple CAS login view."""
def get(self, request): def get(self, request):
ticket = request.GET.get('ticket') ticket = request.GET.get("ticket")
if ticket: if ticket:
# Coming back from CAS with ticket - validate it # Coming back from CAS with ticket - validate it
service_url = request.build_absolute_uri().split('?')[0] # Remove query params service_url = request.build_absolute_uri().split("?")[
0
] # Remove query params
user = authenticate(request=request, ticket=ticket, service=service_url) user = authenticate(request=request, ticket=ticket, service=service_url)
if user: if user:
login(request, user) login(request, user)
return redirect(settings.LOGIN_REDIRECT_URL) return redirect(settings.LOGIN_REDIRECT_URL)
else: else:
return HttpResponse("Authentication failed", status=401) return HttpResponse("Authentication failed", status=401)
else: else:
# No ticket - redirect to CAS # No ticket - redirect to CAS
service_url = request.build_absolute_uri().split('?')[0] # Remove query params service_url = request.build_absolute_uri().split("?")[
0
] # Remove query params
cas_login_url = f"{settings.CAS_SERVER_URL.rstrip('/')}/login?service={urllib.parse.quote(service_url)}" cas_login_url = f"{settings.CAS_SERVER_URL.rstrip('/')}/login?service={urllib.parse.quote(service_url)}"
return redirect(cas_login_url) return redirect(cas_login_url)
class SimpleCASLogoutView(View): class SimpleCASLogoutView(View):
"""Simple CAS logout view.""" """Simple CAS logout view."""
def get(self, request): def get(self, request):
logout(request) logout(request)
# Redirect to CAS logout # Redirect to CAS logout
cas_logout_url = f"{settings.CAS_SERVER_URL.rstrip('/')}/logout" cas_logout_url = f"{settings.CAS_SERVER_URL.rstrip('/')}/logout"
return redirect(cas_logout_url) return redirect(cas_logout_url)
opus_submitter/src/components/FileUpload.vue
+48
View File
@@ -146,6 +146,33 @@
</div> </div>
</div> </div>
<!-- Manual Puzzle Selection (when OCR confidence is low) -->
<div v-if="file.needsManualPuzzleSelection" class="mt-2">
<div class="alert alert-warning alert-sm">
<i class="mdi mdi-alert-circle text-lg"></i>
<div class="flex-1">
<div class="font-medium">Low OCR Confidence</div>
<div class="text-xs">Please select the correct puzzle manually</div>
</div>
</div>
<div class="mt-2">
<select
v-model="file.manualPuzzleSelection"
class="select select-bordered select-sm w-full"
@change="onManualPuzzleSelection(file)"
>
<option value="">Select puzzle...</option>
<option
v-for="puzzle in puzzlesStore.puzzles"
:key="puzzle.id"
:value="puzzle.title"
>
{{ puzzle.title }}
</option>
</select>
</div>
</div>
<!-- Manual OCR trigger for non-auto detected files --> <!-- Manual OCR trigger for non-auto detected files -->
<div v-else-if="!file.ocrProcessing && !file.ocrError && !file.ocrData" class="mt-1"> <div v-else-if="!file.ocrProcessing && !file.ocrError && !file.ocrData" class="mt-1">
<button <button
@@ -335,6 +362,15 @@ const processOCR = async (submissionFile: SubmissionFile) => {
// Force reactivity update // Force reactivity update
await nextTick() await nextTick()
files.value[fileIndex].ocrData = ocrData files.value[fileIndex].ocrData = ocrData
// Check if puzzle confidence is below 80% and needs manual selection
if (ocrData.confidence.puzzle < 0.8) {
files.value[fileIndex].needsManualPuzzleSelection = true
console.log(`Low puzzle confidence (${Math.round(ocrData.confidence.puzzle * 100)}%) for ${submissionFile.file.name}, requiring manual selection`)
} else {
files.value[fileIndex].needsManualPuzzleSelection = false
}
await nextTick() await nextTick()
} catch (error) { } catch (error) {
console.error('OCR processing failed:', error) console.error('OCR processing failed:', error)
@@ -353,4 +389,16 @@ const getConfidenceBadgeClass = (confidence: number): string => {
if (confidence >= 0.6) return 'badge-warning' if (confidence >= 0.6) return 'badge-warning'
return 'badge-error' return 'badge-error'
} }
const onManualPuzzleSelection = (submissionFile: SubmissionFile) => {
// Find the file in the reactive array
const fileIndex = files.value.findIndex(f => f.file === submissionFile.file)
if (fileIndex === -1) return
// Clear the manual selection requirement once user has selected
if (files.value[fileIndex].manualPuzzleSelection) {
files.value[fileIndex].needsManualPuzzleSelection = false
console.log(`Manual puzzle selection: ${submissionFile.file.name} -> ${files.value[fileIndex].manualPuzzleSelection}`)
}
}
</script> </script>
opus_submitter/src/components/SubmissionForm.vue
+33 -6
View File
@@ -23,6 +23,18 @@
<!-- File Upload --> <!-- File Upload -->
<FileUpload v-model="submissionFiles" :puzzles="puzzles" /> <FileUpload v-model="submissionFiles" :puzzles="puzzles" />
<!-- Manual Selection Warning -->
<div v-if="filesNeedingManualSelection.length > 0" class="alert alert-warning">
<i class="mdi mdi-alert-circle text-xl"></i>
<div class="flex-1">
<div class="font-bold">Manual Puzzle Selection Required</div>
<div class="text-sm">
{{ filesNeedingManualSelection.length }} file(s) have low OCR confidence for puzzle names.
Please select the correct puzzle for each file before submitting.
</div>
</div>
</div>
<!-- Notes --> <!-- Notes -->
<div class="form-control"> <div class="form-control">
@@ -64,10 +76,14 @@
<button <button
type="submit" type="submit"
class="btn btn-primary" class="btn btn-primary"
:disabled="isSubmitting" :disabled="!canSubmit"
> >
<span v-if="isSubmitting" class="loading loading-spinner loading-sm"></span> <span v-if="isSubmitting" class="loading loading-spinner loading-sm"></span>
{{ isSubmitting ? 'Submitting...' : 'Submit Solution' }} <span v-if="isSubmitting">Submitting...</span>
<span v-else-if="filesNeedingManualSelection.length > 0">
Select Puzzles ({{ filesNeedingManualSelection.length }} remaining)
</span>
<span v-else>Submit Solution</span>
</button> </button>
</div> </div>
</form> </form>
@@ -100,8 +116,12 @@ const isSubmitting = ref(false)
const notesLength = computed(() => notes.value.length) const notesLength = computed(() => notes.value.length)
const canSubmit = computed(() => { const canSubmit = computed(() => {
return submissionFiles.value.length > 0 && const hasFiles = submissionFiles.value.length > 0
!isSubmitting.value const noManualSelectionNeeded = !submissionFiles.value.some(file => file.needsManualPuzzleSelection)
return hasFiles &&
!isSubmitting.value &&
noManualSelectionNeeded
}) })
// Group files by detected puzzle // Group files by detected puzzle
@@ -109,8 +129,10 @@ const responsesByPuzzle = computed(() => {
const grouped: Record<string, { puzzle: SteamCollectionItem | null, files: SubmissionFile[] }> = {} const grouped: Record<string, { puzzle: SteamCollectionItem | null, files: SubmissionFile[] }> = {}
submissionFiles.value.forEach(file => { submissionFiles.value.forEach(file => {
if (file.ocrData?.puzzle) { // Use manual puzzle selection if available, otherwise fall back to OCR
const puzzleName = file.ocrData.puzzle const puzzleName = file.manualPuzzleSelection || file.ocrData?.puzzle
if (puzzleName) {
if (!grouped[puzzleName]) { if (!grouped[puzzleName]) {
grouped[puzzleName] = { grouped[puzzleName] = {
puzzle: props.findPuzzleByName(puzzleName), puzzle: props.findPuzzleByName(puzzleName),
@@ -124,6 +146,11 @@ const responsesByPuzzle = computed(() => {
return grouped return grouped
}) })
// Count files that need manual puzzle selection
const filesNeedingManualSelection = computed(() => {
return submissionFiles.value.filter(file => file.needsManualPuzzleSelection)
})
// Check if any OCR confidence is below 50% // Check if any OCR confidence is below 50%
const hasLowConfidence = computed(() => { const hasLowConfidence = computed(() => {
return submissionFiles.value.some(file => { return submissionFiles.value.some(file => {
opus_submitter/src/services/apiService.ts
+4 -2
View File
@@ -238,8 +238,10 @@ export const submissionHelpers = {
}> = {} }> = {}
files.forEach(file => { files.forEach(file => {
if (file.ocrData?.puzzle) { // Use manual puzzle selection if available, otherwise fall back to OCR
const puzzleName = file.ocrData.puzzle const puzzleName = file.manualPuzzleSelection || file.ocrData?.puzzle
if (puzzleName) {
if (!responsesByPuzzle[puzzleName]) { if (!responsesByPuzzle[puzzleName]) {
responsesByPuzzle[puzzleName] = { responsesByPuzzle[puzzleName] = {
puzzle: puzzleHelpers.findPuzzleByName(puzzles, puzzleName), puzzle: puzzleHelpers.findPuzzleByName(puzzles, puzzleName),
opus_submitter/src/types/index.ts
+2
View File
@@ -46,6 +46,8 @@ export interface SubmissionFile {
ocrProcessing?: boolean ocrProcessing?: boolean
ocrError?: string ocrError?: string
original_filename?: string original_filename?: string
manualPuzzleSelection?: string
needsManualPuzzleSelection?: boolean
} }
export interface PuzzleResponse { export interface PuzzleResponse {
opus_submitter/static_source/vite/assets/main-B14l8Jy0.js
+21
View File
File diff suppressed because one or more lines are too long
opus_submitter/static_source/vite/assets/main-DeQiP-Az.css → opus_submitter/static_source/vite/assets/main-COx9N9qO.css
+1 -1
View File
File diff suppressed because one or more lines are too long
opus_submitter/static_source/vite/assets/main-fzs-6OUY.js
-21
View File
File diff suppressed because one or more lines are too long
opus_submitter/static_source/vite/manifest.json
+2 -2
View File
@@ -16,12 +16,12 @@
"src": "node_modules/.pnpm/@mdi+font@7.4.47/node_modules/@mdi/font/fonts/materialdesignicons-webfont.woff2" "src": "node_modules/.pnpm/@mdi+font@7.4.47/node_modules/@mdi/font/fonts/materialdesignicons-webfont.woff2"
}, },
"src/main.ts": { "src/main.ts": {
"file": "assets/main-fzs-6OUY.js", "file": "assets/main-B14l8Jy0.js",
"name": "main", "name": "main",
"src": "src/main.ts", "src": "src/main.ts",
"isEntry": true, "isEntry": true,
"css": [ "css": [
"assets/main-DeQiP-Az.css" "assets/main-COx9N9qO.css"
], ],
"assets": [ "assets": [
"assets/materialdesignicons-webfont-CSr8KVlo.eot", "assets/materialdesignicons-webfont-CSr8KVlo.eot",
opus_submitter/submissions/admin.py
+158 -82
View File
@@ -2,8 +2,12 @@ from django.contrib import admin
from django.utils.html import format_html from django.utils.html import format_html
from django.utils import timezone from django.utils import timezone
from .models import ( from .models import (
SteamAPIKey, SteamCollection, SteamCollectionItem, SteamAPIKey,
Submission, PuzzleResponse, SubmissionFile SteamCollection,
SteamCollectionItem,
Submission,
PuzzleResponse,
SubmissionFile,
) )
@@ -151,7 +155,14 @@ class SubmissionFileInline(admin.TabularInline):
model = SubmissionFile model = SubmissionFile
extra = 0 extra = 0
readonly_fields = ["file_size", "content_type", "ocr_processed", "created_at"] readonly_fields = ["file_size", "content_type", "ocr_processed", "created_at"]
fields = ["file", "original_filename", "file_size", "content_type", "ocr_processed", "ocr_error"] fields = [
"file",
"original_filename",
"file_size",
"content_type",
"ocr_processed",
"ocr_error",
]
class PuzzleResponseInline(admin.TabularInline): class PuzzleResponseInline(admin.TabularInline):
@@ -159,43 +170,73 @@ class PuzzleResponseInline(admin.TabularInline):
extra = 0 extra = 0
readonly_fields = ["created_at", "updated_at"] readonly_fields = ["created_at", "updated_at"]
fields = [ fields = [
"puzzle", "puzzle_name", "cost", "cycles", "area", "puzzle",
"needs_manual_validation", "ocr_confidence_cost", "ocr_confidence_cycles", "ocr_confidence_area" "puzzle_name",
"cost",
"cycles",
"area",
"needs_manual_validation",
"ocr_confidence_cost",
"ocr_confidence_cycles",
"ocr_confidence_area",
] ]
@admin.register(Submission) @admin.register(Submission)
class SubmissionAdmin(admin.ModelAdmin): class SubmissionAdmin(admin.ModelAdmin):
list_display = [ list_display = [
"id", "user", "total_responses", "needs_validation", "id",
"manual_validation_requested", "is_validated", "created_at" "user",
"total_responses",
"needs_validation",
"manual_validation_requested",
"is_validated",
"created_at",
] ]
list_filter = [ list_filter = [
"is_validated", "manual_validation_requested", "created_at", "updated_at" "is_validated",
"manual_validation_requested",
"created_at",
"updated_at",
] ]
search_fields = ["id", "user__username", "notes"] search_fields = ["id", "user__username", "notes"]
readonly_fields = ["id", "created_at", "updated_at", "total_responses", "needs_validation"] readonly_fields = [
"id",
"created_at",
"updated_at",
"total_responses",
"needs_validation",
]
inlines = [PuzzleResponseInline] inlines = [PuzzleResponseInline]
fieldsets = ( fieldsets = (
("Basic Information", { ("Basic Information", {"fields": ("id", "user", "notes")}),
"fields": ("id", "user", "notes") (
}), "Validation",
("Validation", { {
"fields": ("manual_validation_requested", "is_validated", "validated_by", "validated_at") "fields": (
}), "manual_validation_requested",
("Statistics", { "is_validated",
"fields": ("total_responses", "needs_validation"), "validated_by",
"classes": ("collapse",) "validated_at",
}), )
("Timestamps", { },
"fields": ("created_at", "updated_at"), ),
"classes": ("collapse",) (
}), "Statistics",
{
"fields": ("total_responses", "needs_validation"),
"classes": ("collapse",),
},
),
(
"Timestamps",
{"fields": ("created_at", "updated_at"), "classes": ("collapse",)},
),
) )
actions = ["mark_as_validated"] actions = ["mark_as_validated"]
def mark_as_validated(self, request, queryset): def mark_as_validated(self, request, queryset):
"""Mark selected submissions as validated""" """Mark selected submissions as validated"""
updated = 0 updated = 0
@@ -208,59 +249,82 @@ class SubmissionAdmin(admin.ModelAdmin):
# Also mark all responses as not needing validation # Also mark all responses as not needing validation
submission.responses.update(needs_manual_validation=False) submission.responses.update(needs_manual_validation=False)
updated += 1 updated += 1
self.message_user(request, f"{updated} submissions marked as validated.") self.message_user(request, f"{updated} submissions marked as validated.")
mark_as_validated.short_description = "Mark selected submissions as validated" mark_as_validated.short_description = "Mark selected submissions as validated"
@admin.register(PuzzleResponse) @admin.register(PuzzleResponse)
class PuzzleResponseAdmin(admin.ModelAdmin): class PuzzleResponseAdmin(admin.ModelAdmin):
list_display = [ list_display = [
"puzzle_name", "submission", "puzzle", "cost", "cycles", "area", "puzzle_name",
"needs_manual_validation", "created_at" "submission",
] "puzzle",
list_filter = [ "cost",
"needs_manual_validation", "puzzle__collection", "created_at" "cycles",
"area",
"needs_manual_validation",
"created_at",
] ]
list_filter = ["needs_manual_validation", "puzzle__collection", "created_at"]
search_fields = [ search_fields = [
"puzzle_name", "submission__id", "puzzle__title", "puzzle_name",
"cost", "cycles", "area" "submission__id",
"puzzle__title",
"cost",
"cycles",
"area",
] ]
readonly_fields = ["created_at", "updated_at"] readonly_fields = ["created_at", "updated_at"]
inlines = [SubmissionFileInline] inlines = [SubmissionFileInline]
fieldsets = ( fieldsets = (
("Basic Information", { ("Basic Information", {"fields": ("submission", "puzzle", "puzzle_name")}),
"fields": ("submission", "puzzle", "puzzle_name") (
}), "OCR Data",
("OCR Data", { {
"fields": ("cost", "cycles", "area", "ocr_confidence_cost", "ocr_confidence_cycles", "ocr_confidence_area") "fields": (
}), "cost",
("Validation", { "cycles",
"fields": ( "area",
"needs_manual_validation", "ocr_confidence_cost",
"validated_cost", "validated_cycles", "validated_area" "ocr_confidence_cycles",
) "ocr_confidence_area",
}), )
("Timestamps", { },
"fields": ("created_at", "updated_at"), ),
"classes": ("collapse",) (
}), "Validation",
{
"fields": (
"needs_manual_validation",
"validated_cost",
"validated_cycles",
"validated_area",
)
},
),
(
"Timestamps",
{"fields": ("created_at", "updated_at"), "classes": ("collapse",)},
),
) )
actions = ["mark_for_validation", "clear_validation_flag"] actions = ["mark_for_validation", "clear_validation_flag"]
def mark_for_validation(self, request, queryset): def mark_for_validation(self, request, queryset):
"""Mark selected responses as needing validation""" """Mark selected responses as needing validation"""
updated = queryset.update(needs_manual_validation=True) updated = queryset.update(needs_manual_validation=True)
self.message_user(request, f"{updated} responses marked for validation.") self.message_user(request, f"{updated} responses marked for validation.")
def clear_validation_flag(self, request, queryset): def clear_validation_flag(self, request, queryset):
"""Clear validation flag for selected responses""" """Clear validation flag for selected responses"""
updated = queryset.update(needs_manual_validation=False) updated = queryset.update(needs_manual_validation=False)
self.message_user(request, f"{updated} responses cleared from validation queue.") self.message_user(
request, f"{updated} responses cleared from validation queue."
)
mark_for_validation.short_description = "Mark as needing validation" mark_for_validation.short_description = "Mark as needing validation"
clear_validation_flag.short_description = "Clear validation flag" clear_validation_flag.short_description = "Clear validation flag"
@@ -268,37 +332,49 @@ class PuzzleResponseAdmin(admin.ModelAdmin):
@admin.register(SubmissionFile) @admin.register(SubmissionFile)
class SubmissionFileAdmin(admin.ModelAdmin): class SubmissionFileAdmin(admin.ModelAdmin):
list_display = [ list_display = [
"original_filename", "response", "file_size_display", "original_filename",
"content_type", "ocr_processed", "created_at" "response",
] "file_size_display",
list_filter = [ "content_type",
"content_type", "ocr_processed", "created_at" "ocr_processed",
"created_at",
] ]
list_filter = ["content_type", "ocr_processed", "created_at"]
search_fields = [ search_fields = [
"original_filename", "response__puzzle_name", "original_filename",
"response__submission__id" "response__puzzle_name",
"response__submission__id",
] ]
readonly_fields = [ readonly_fields = [
"file_size", "content_type", "ocr_processed", "file_size",
"created_at", "updated_at", "file_url" "content_type",
"ocr_processed",
"created_at",
"updated_at",
"file_url",
] ]
fieldsets = ( fieldsets = (
("File Information", { (
"fields": ("file", "original_filename", "file_size", "content_type", "file_url") "File Information",
}), {
("OCR Processing", { "fields": (
"fields": ("ocr_processed", "ocr_raw_data", "ocr_error") "file",
}), "original_filename",
("Relationships", { "file_size",
"fields": ("response",) "content_type",
}), "file_url",
("Timestamps", { )
"fields": ("created_at", "updated_at"), },
"classes": ("collapse",) ),
}), ("OCR Processing", {"fields": ("ocr_processed", "ocr_raw_data", "ocr_error")}),
("Relationships", {"fields": ("response",)}),
(
"Timestamps",
{"fields": ("created_at", "updated_at"), "classes": ("collapse",)},
),
) )
def file_size_display(self, obj): def file_size_display(self, obj):
"""Display file size in human readable format""" """Display file size in human readable format"""
if obj.file_size < 1024: if obj.file_size < 1024:
@@ -307,5 +383,5 @@ class SubmissionFileAdmin(admin.ModelAdmin):
return f"{obj.file_size / 1024:.1f} KB" return f"{obj.file_size / 1024:.1f} KB"
else: else:
return f"{obj.file_size / (1024 * 1024):.1f} MB" return f"{obj.file_size / (1024 * 1024):.1f} MB"
file_size_display.short_description = "File Size" file_size_display.short_description = "File Size"
opus_submitter/submissions/api.py
+16 -6
View File
@@ -69,17 +69,27 @@ def create_submission(
with transaction.atomic(): with transaction.atomic():
# Check if any confidence score is below 50% to auto-request validation # Check if any confidence score is below 50% to auto-request validation
auto_request_validation = any( auto_request_validation = any(
(response_data.ocr_confidence_cost is not None and response_data.ocr_confidence_cost < 0.5) or (
(response_data.ocr_confidence_cycles is not None and response_data.ocr_confidence_cycles < 0.5) or response_data.ocr_confidence_cost is not None
(response_data.ocr_confidence_area is not None and response_data.ocr_confidence_area < 0.5) and response_data.ocr_confidence_cost < 0.5
)
or (
response_data.ocr_confidence_cycles is not None
and response_data.ocr_confidence_cycles < 0.5
)
or (
response_data.ocr_confidence_area is not None
and response_data.ocr_confidence_area < 0.5
)
for response_data in data.responses for response_data in data.responses
) )
# Create the submission # Create the submission
submission = Submission.objects.create( submission = Submission.objects.create(
user=request.user if request.user.is_authenticated else None, user=request.user if request.user.is_authenticated else None,
notes=data.notes, notes=data.notes,
manual_validation_requested=data.manual_validation_requested or auto_request_validation, manual_validation_requested=data.manual_validation_requested
or auto_request_validation,
) )
file_index = 0 file_index = 0
@@ -100,7 +110,7 @@ def create_submission(
cost=response_data.cost, cost=response_data.cost,
cycles=response_data.cycles, cycles=response_data.cycles,
area=response_data.area, area=response_data.area,
needs_manual_validation=response_data.needs_manual_validation, needs_manual_validation=data.manual_validation_requested,
ocr_confidence_cost=response_data.ocr_confidence_cost, ocr_confidence_cost=response_data.ocr_confidence_cost,
ocr_confidence_cycles=response_data.ocr_confidence_cycles, ocr_confidence_cycles=response_data.ocr_confidence_cycles,
ocr_confidence_area=response_data.ocr_confidence_area, ocr_confidence_area=response_data.ocr_confidence_area,
opus_submitter/submissions/apps.py
+2 -2
View File
@@ -2,5 +2,5 @@ from django.apps import AppConfig
class SubmissionsConfig(AppConfig): class SubmissionsConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField' default_auto_field = "django.db.models.BigAutoField"
name = 'submissions' name = "submissions"
opus_submitter/submissions/management/commands/fetch_steam_collection.py
+34 -29
View File
@@ -8,40 +8,39 @@ from submissions.models import SteamCollection
class Command(BaseCommand): class Command(BaseCommand):
help = 'Fetch Steam Workshop collection data and save to database' help = "Fetch Steam Workshop collection data and save to database"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument("url", type=str, help="Steam Workshop collection URL")
parser.add_argument( parser.add_argument(
'url', "--api-key",
type=str, type=str,
help='Steam Workshop collection URL' help="Steam API key (optional, can also be set via STEAM_API_KEY environment variable)",
) )
parser.add_argument( parser.add_argument(
'--api-key', "--force",
type=str, action="store_true",
help='Steam API key (optional, can also be set via STEAM_API_KEY environment variable)' help="Force refetch even if collection already exists",
) )
parser.add_argument(
'--force',
action='store_true',
help='Force refetch even if collection already exists'
)
def handle(self, *args, **options): def handle(self, *args, **options):
url = options['url'] url = options["url"]
api_key = options.get('api_key') api_key = options.get("api_key")
force = options['force'] force = options["force"]
self.stdout.write(f"Fetching Steam collection from: {url}") self.stdout.write(f"Fetching Steam collection from: {url}")
try: try:
# Check if collection already exists # Check if collection already exists
from submissions.utils import SteamCollectionFetcher from submissions.utils import SteamCollectionFetcher
fetcher = SteamCollectionFetcher(api_key) fetcher = SteamCollectionFetcher(api_key)
collection_id = fetcher.extract_collection_id(url) collection_id = fetcher.extract_collection_id(url)
if collection_id and not force: if collection_id and not force:
existing = SteamCollection.objects.filter(steam_id=collection_id).first() existing = SteamCollection.objects.filter(
steam_id=collection_id
).first()
if existing: if existing:
self.stdout.write( self.stdout.write(
self.style.WARNING( self.style.WARNING(
@@ -50,10 +49,10 @@ class Command(BaseCommand):
) )
) )
return return
# Fetch and create/update collection # Fetch and create/update collection
collection, created = create_or_update_collection(url) collection, created = create_or_update_collection(url)
if created: if created:
self.stdout.write( self.stdout.write(
self.style.SUCCESS( self.style.SUCCESS(
@@ -66,27 +65,33 @@ class Command(BaseCommand):
f"Successfully updated collection: {collection.title} (ID: {collection.id})" f"Successfully updated collection: {collection.title} (ID: {collection.id})"
) )
) )
# Display collection info # Display collection info
self.stdout.write("\nCollection Details:") self.stdout.write("\nCollection Details:")
self.stdout.write(f" Steam ID: {collection.steam_id}") self.stdout.write(f" Steam ID: {collection.steam_id}")
self.stdout.write(f" Title: {collection.title}") self.stdout.write(f" Title: {collection.title}")
self.stdout.write(f" Author: {collection.author_name or 'Unknown'}") self.stdout.write(f" Author: {collection.author_name or 'Unknown'}")
self.stdout.write(f" Description: {collection.description[:100]}{'...' if len(collection.description) > 100 else ''}") self.stdout.write(
f" Description: {collection.description[:100]}{'...' if len(collection.description) > 100 else ''}"
)
self.stdout.write(f" Total Items: {collection.total_items}") self.stdout.write(f" Total Items: {collection.total_items}")
self.stdout.write(f" Unique Visitors: {collection.unique_visitors}") self.stdout.write(f" Unique Visitors: {collection.unique_visitors}")
self.stdout.write(f" Current Favorites: {collection.current_favorites}") self.stdout.write(f" Current Favorites: {collection.current_favorites}")
self.stdout.write(f" Total Favorites: {collection.total_favorites}") self.stdout.write(f" Total Favorites: {collection.total_favorites}")
if collection.items.exists(): if collection.items.exists():
self.stdout.write(f"\nCollection Items ({collection.items.count()}):") self.stdout.write(f"\nCollection Items ({collection.items.count()}):")
for item in collection.items.all()[:10]: # Show first 10 items for item in collection.items.all()[:10]: # Show first 10 items
self.stdout.write(f" - {item.title} (Steam ID: {item.steam_item_id})") self.stdout.write(
f" - {item.title} (Steam ID: {item.steam_item_id})"
)
if collection.items.count() > 10: if collection.items.count() > 10:
self.stdout.write(f" ... and {collection.items.count() - 10} more items") self.stdout.write(
f" ... and {collection.items.count() - 10} more items"
)
else: else:
self.stdout.write("\nNo items found in collection.") self.stdout.write("\nNo items found in collection.")
except Exception as e: except Exception as e:
raise CommandError(f"Failed to fetch collection: {e}") raise CommandError(f"Failed to fetch collection: {e}")
opus_submitter/submissions/migrations/0001_initial.py
+191 -44
View File
@@ -5,68 +5,215 @@ from django.db import migrations, models
class Migration(migrations.Migration): class Migration(migrations.Migration):
initial = True initial = True
dependencies = [ dependencies = []
]
operations = [ operations = [
migrations.CreateModel( migrations.CreateModel(
name='Collection', name="Collection",
fields=[ fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), (
('url', models.URLField()), "id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("url", models.URLField()),
], ],
), ),
migrations.CreateModel( migrations.CreateModel(
name='SteamCollection', name="SteamCollection",
fields=[ fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), (
('steam_id', models.CharField(help_text='Steam collection ID from URL', max_length=50, unique=True)), "id",
('url', models.URLField(help_text='Full Steam Workshop collection URL')), models.BigAutoField(
('title', models.CharField(blank=True, help_text='Collection title', max_length=255)), auto_created=True,
('description', models.TextField(blank=True, help_text='Collection description')), primary_key=True,
('author_name', models.CharField(blank=True, help_text='Steam username of collection creator', max_length=100)), serialize=False,
('author_steam_id', models.CharField(blank=True, help_text='Steam ID of collection creator', max_length=50)), verbose_name="ID",
('total_items', models.PositiveIntegerField(default=0, help_text='Number of items in collection')), ),
('unique_visitors', models.PositiveIntegerField(default=0, help_text='Number of unique visitors')), ),
('current_favorites', models.PositiveIntegerField(default=0, help_text='Current number of favorites')), (
('total_favorites', models.PositiveIntegerField(default=0, help_text='Total unique favorites')), "steam_id",
('steam_created_date', models.DateTimeField(blank=True, help_text='When collection was created on Steam', null=True)), models.CharField(
('steam_updated_date', models.DateTimeField(blank=True, help_text='When collection was last updated on Steam', null=True)), help_text="Steam collection ID from URL",
('created_at', models.DateTimeField(auto_now_add=True)), max_length=50,
('updated_at', models.DateTimeField(auto_now=True)), unique=True,
('last_fetched', models.DateTimeField(blank=True, help_text='When data was last fetched from Steam', null=True)), ),
('is_active', models.BooleanField(default=True, help_text='Whether this collection is actively tracked')), ),
('fetch_error', models.TextField(blank=True, help_text='Last error encountered when fetching data')), (
"url",
models.URLField(help_text="Full Steam Workshop collection URL"),
),
(
"title",
models.CharField(
blank=True, help_text="Collection title", max_length=255
),
),
(
"description",
models.TextField(blank=True, help_text="Collection description"),
),
(
"author_name",
models.CharField(
blank=True,
help_text="Steam username of collection creator",
max_length=100,
),
),
(
"author_steam_id",
models.CharField(
blank=True,
help_text="Steam ID of collection creator",
max_length=50,
),
),
(
"total_items",
models.PositiveIntegerField(
default=0, help_text="Number of items in collection"
),
),
(
"unique_visitors",
models.PositiveIntegerField(
default=0, help_text="Number of unique visitors"
),
),
(
"current_favorites",
models.PositiveIntegerField(
default=0, help_text="Current number of favorites"
),
),
(
"total_favorites",
models.PositiveIntegerField(
default=0, help_text="Total unique favorites"
),
),
(
"steam_created_date",
models.DateTimeField(
blank=True,
help_text="When collection was created on Steam",
null=True,
),
),
(
"steam_updated_date",
models.DateTimeField(
blank=True,
help_text="When collection was last updated on Steam",
null=True,
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"last_fetched",
models.DateTimeField(
blank=True,
help_text="When data was last fetched from Steam",
null=True,
),
),
(
"is_active",
models.BooleanField(
default=True,
help_text="Whether this collection is actively tracked",
),
),
(
"fetch_error",
models.TextField(
blank=True,
help_text="Last error encountered when fetching data",
),
),
], ],
options={ options={
'verbose_name': 'Steam Collection', "verbose_name": "Steam Collection",
'verbose_name_plural': 'Steam Collections', "verbose_name_plural": "Steam Collections",
'ordering': ['-created_at'], "ordering": ["-created_at"],
}, },
), ),
migrations.CreateModel( migrations.CreateModel(
name='SteamCollectionItem', name="SteamCollectionItem",
fields=[ fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), (
('steam_item_id', models.CharField(help_text='Steam Workshop item ID', max_length=50)), "id",
('title', models.CharField(blank=True, help_text='Item title', max_length=255)), models.BigAutoField(
('author_name', models.CharField(blank=True, help_text='Steam username of item creator', max_length=100)), auto_created=True,
('author_steam_id', models.CharField(blank=True, help_text='Steam ID of item creator', max_length=50)), primary_key=True,
('description', models.TextField(blank=True, help_text='Item description')), serialize=False,
('tags', models.JSONField(blank=True, default=list, help_text='Item tags as JSON array')), verbose_name="ID",
('order_index', models.PositiveIntegerField(default=0, help_text='Order of item in collection')), ),
('created_at', models.DateTimeField(auto_now_add=True)), ),
('updated_at', models.DateTimeField(auto_now=True)), (
('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='submissions.steamcollection')), "steam_item_id",
models.CharField(help_text="Steam Workshop item ID", max_length=50),
),
(
"title",
models.CharField(
blank=True, help_text="Item title", max_length=255
),
),
(
"author_name",
models.CharField(
blank=True,
help_text="Steam username of item creator",
max_length=100,
),
),
(
"author_steam_id",
models.CharField(
blank=True, help_text="Steam ID of item creator", max_length=50
),
),
(
"description",
models.TextField(blank=True, help_text="Item description"),
),
(
"tags",
models.JSONField(
blank=True, default=list, help_text="Item tags as JSON array"
),
),
(
"order_index",
models.PositiveIntegerField(
default=0, help_text="Order of item in collection"
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"collection",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="items",
to="submissions.steamcollection",
),
),
], ],
options={ options={
'verbose_name': 'Steam Collection Item', "verbose_name": "Steam Collection Item",
'verbose_name_plural': 'Steam Collection Items', "verbose_name_plural": "Steam Collection Items",
'ordering': ['collection', 'order_index'], "ordering": ["collection", "order_index"],
'unique_together': {('collection', 'steam_item_id')}, "unique_together": {("collection", "steam_item_id")},
}, },
), ),
] ]
opus_submitter/submissions/migrations/0002_delete_collection.py
+2 -3
View File
@@ -4,13 +4,12 @@ from django.db import migrations
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('submissions', '0001_initial'), ("submissions", "0001_initial"),
] ]
operations = [ operations = [
migrations.DeleteModel( migrations.DeleteModel(
name='Collection', name="Collection",
), ),
] ]
opus_submitter/submissions/migrations/0003_steamapikey.py
+52 -14
View File
@@ -4,28 +4,66 @@ from django.db import migrations, models
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('submissions', '0002_delete_collection'), ("submissions", "0002_delete_collection"),
] ]
operations = [ operations = [
migrations.CreateModel( migrations.CreateModel(
name='SteamAPIKey', name="SteamAPIKey",
fields=[ fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), (
('name', models.CharField(help_text="Descriptive name for this API key (e.g., 'Production Key', 'Development Key')", max_length=100, unique=True)), "id",
('api_key', models.CharField(help_text='Steam Web API key from https://steamcommunity.com/dev/apikey', max_length=64)), models.BigAutoField(
('is_active', models.BooleanField(default=True, help_text='Whether this API key should be used')), auto_created=True,
('description', models.TextField(blank=True, help_text='Optional description or notes about this API key')), primary_key=True,
('created_at', models.DateTimeField(auto_now_add=True)), serialize=False,
('updated_at', models.DateTimeField(auto_now=True)), verbose_name="ID",
('last_used', models.DateTimeField(blank=True, help_text='When this API key was last used', null=True)), ),
),
(
"name",
models.CharField(
help_text="Descriptive name for this API key (e.g., 'Production Key', 'Development Key')",
max_length=100,
unique=True,
),
),
(
"api_key",
models.CharField(
help_text="Steam Web API key from https://steamcommunity.com/dev/apikey",
max_length=64,
),
),
(
"is_active",
models.BooleanField(
default=True, help_text="Whether this API key should be used"
),
),
(
"description",
models.TextField(
blank=True,
help_text="Optional description or notes about this API key",
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"last_used",
models.DateTimeField(
blank=True,
help_text="When this API key was last used",
null=True,
),
),
], ],
options={ options={
'verbose_name': 'Steam API Key', "verbose_name": "Steam API Key",
'verbose_name_plural': 'Steam API Keys', "verbose_name_plural": "Steam API Keys",
'ordering': ['-is_active', 'name'], "ordering": ["-is_active", "name"],
}, },
), ),
] ]
opus_submitter/submissions/migrations/0004_submission_puzzleresponse_submissionfile.py
+218 -48
View File
@@ -8,75 +8,245 @@ from django.db import migrations, models
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('submissions', '0003_steamapikey'), ("submissions", "0003_steamapikey"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL), migrations.swappable_dependency(settings.AUTH_USER_MODEL),
] ]
operations = [ operations = [
migrations.CreateModel( migrations.CreateModel(
name='Submission', name="Submission",
fields=[ fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), (
('notes', models.TextField(blank=True, help_text='Optional notes about the submission')), "id",
('is_validated', models.BooleanField(default=False, help_text='Whether this submission has been manually validated')), models.UUIDField(
('validated_at', models.DateTimeField(blank=True, help_text='When this submission was validated', null=True)), default=uuid.uuid4,
('created_at', models.DateTimeField(auto_now_add=True)), editable=False,
('updated_at', models.DateTimeField(auto_now=True)), primary_key=True,
('user', models.ForeignKey(blank=True, help_text='User who made the submission (null for anonymous)', null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), serialize=False,
('validated_by', models.ForeignKey(blank=True, help_text='Admin user who validated this submission', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='validated_submissions', to=settings.AUTH_USER_MODEL)), ),
),
(
"notes",
models.TextField(
blank=True, help_text="Optional notes about the submission"
),
),
(
"is_validated",
models.BooleanField(
default=False,
help_text="Whether this submission has been manually validated",
),
),
(
"validated_at",
models.DateTimeField(
blank=True,
help_text="When this submission was validated",
null=True,
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"user",
models.ForeignKey(
blank=True,
help_text="User who made the submission (null for anonymous)",
null=True,
on_delete=django.db.models.deletion.CASCADE,
to=settings.AUTH_USER_MODEL,
),
),
(
"validated_by",
models.ForeignKey(
blank=True,
help_text="Admin user who validated this submission",
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="validated_submissions",
to=settings.AUTH_USER_MODEL,
),
),
], ],
options={ options={
'verbose_name': 'Submission', "verbose_name": "Submission",
'verbose_name_plural': 'Submissions', "verbose_name_plural": "Submissions",
'ordering': ['-created_at'], "ordering": ["-created_at"],
}, },
), ),
migrations.CreateModel( migrations.CreateModel(
name='PuzzleResponse', name="PuzzleResponse",
fields=[ fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), (
('puzzle_name', models.CharField(help_text='Puzzle name as detected by OCR', max_length=255)), "id",
('cost', models.CharField(blank=True, help_text='Cost value from OCR', max_length=20)), models.BigAutoField(
('cycles', models.CharField(blank=True, help_text='Cycles value from OCR', max_length=20)), auto_created=True,
('area', models.CharField(blank=True, help_text='Area value from OCR', max_length=20)), primary_key=True,
('needs_manual_validation', models.BooleanField(default=False, help_text='Whether OCR failed and manual validation is needed')), serialize=False,
('ocr_confidence_score', models.FloatField(blank=True, help_text='OCR confidence score (0.0 to 1.0)', null=True)), verbose_name="ID",
('validated_cost', models.CharField(blank=True, help_text='Manually validated cost value', max_length=20)), ),
('validated_cycles', models.CharField(blank=True, help_text='Manually validated cycles value', max_length=20)), ),
('validated_area', models.CharField(blank=True, help_text='Manually validated area value', max_length=20)), (
('created_at', models.DateTimeField(auto_now_add=True)), "puzzle_name",
('updated_at', models.DateTimeField(auto_now=True)), models.CharField(
('puzzle', models.ForeignKey(help_text='The puzzle this response is for', on_delete=django.db.models.deletion.CASCADE, related_name='responses', to='submissions.steamcollectionitem')), help_text="Puzzle name as detected by OCR", max_length=255
('submission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='responses', to='submissions.submission')), ),
),
(
"cost",
models.CharField(
blank=True, help_text="Cost value from OCR", max_length=20
),
),
(
"cycles",
models.CharField(
blank=True, help_text="Cycles value from OCR", max_length=20
),
),
(
"area",
models.CharField(
blank=True, help_text="Area value from OCR", max_length=20
),
),
(
"needs_manual_validation",
models.BooleanField(
default=False,
help_text="Whether OCR failed and manual validation is needed",
),
),
(
"ocr_confidence_score",
models.FloatField(
blank=True,
help_text="OCR confidence score (0.0 to 1.0)",
null=True,
),
),
(
"validated_cost",
models.CharField(
blank=True,
help_text="Manually validated cost value",
max_length=20,
),
),
(
"validated_cycles",
models.CharField(
blank=True,
help_text="Manually validated cycles value",
max_length=20,
),
),
(
"validated_area",
models.CharField(
blank=True,
help_text="Manually validated area value",
max_length=20,
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"puzzle",
models.ForeignKey(
help_text="The puzzle this response is for",
on_delete=django.db.models.deletion.CASCADE,
related_name="responses",
to="submissions.steamcollectionitem",
),
),
(
"submission",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="responses",
to="submissions.submission",
),
),
], ],
options={ options={
'verbose_name': 'Puzzle Response', "verbose_name": "Puzzle Response",
'verbose_name_plural': 'Puzzle Responses', "verbose_name_plural": "Puzzle Responses",
'ordering': ['submission', 'puzzle__order_index'], "ordering": ["submission", "puzzle__order_index"],
'unique_together': {('submission', 'puzzle')}, "unique_together": {("submission", "puzzle")},
}, },
), ),
migrations.CreateModel( migrations.CreateModel(
name='SubmissionFile', name="SubmissionFile",
fields=[ fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), (
('file', models.FileField(help_text='Uploaded file (image/gif)', upload_to=submissions.models.submission_file_upload_path)), "id",
('original_filename', models.CharField(help_text='Original filename as uploaded by user', max_length=255)), models.BigAutoField(
('file_size', models.PositiveIntegerField(help_text='File size in bytes')), auto_created=True,
('content_type', models.CharField(help_text='MIME type of the file', max_length=100)), primary_key=True,
('ocr_processed', models.BooleanField(default=False, help_text='Whether OCR has been processed for this file')), serialize=False,
('ocr_raw_data', models.JSONField(blank=True, help_text='Raw OCR data as JSON', null=True)), verbose_name="ID",
('ocr_error', models.TextField(blank=True, help_text='OCR processing error message')), ),
('created_at', models.DateTimeField(auto_now_add=True)), ),
('updated_at', models.DateTimeField(auto_now=True)), (
('response', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='submissions.puzzleresponse')), "file",
models.FileField(
help_text="Uploaded file (image/gif)",
upload_to=submissions.models.submission_file_upload_path,
),
),
(
"original_filename",
models.CharField(
help_text="Original filename as uploaded by user",
max_length=255,
),
),
(
"file_size",
models.PositiveIntegerField(help_text="File size in bytes"),
),
(
"content_type",
models.CharField(help_text="MIME type of the file", max_length=100),
),
(
"ocr_processed",
models.BooleanField(
default=False,
help_text="Whether OCR has been processed for this file",
),
),
(
"ocr_raw_data",
models.JSONField(
blank=True, help_text="Raw OCR data as JSON", null=True
),
),
(
"ocr_error",
models.TextField(
blank=True, help_text="OCR processing error message"
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"response",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="files",
to="submissions.puzzleresponse",
),
),
], ],
options={ options={
'verbose_name': 'Submission File', "verbose_name": "Submission File",
'verbose_name_plural': 'Submission Files', "verbose_name_plural": "Submission Files",
'ordering': ['response', 'created_at'], "ordering": ["response", "created_at"],
}, },
), ),
] ]
opus_submitter/submissions/migrations/0005_alter_submission_notes.py
+6 -5
View File
@@ -4,15 +4,16 @@ from django.db import migrations, models
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('submissions', '0004_submission_puzzleresponse_submissionfile'), ("submissions", "0004_submission_puzzleresponse_submissionfile"),
] ]
operations = [ operations = [
migrations.AlterField( migrations.AlterField(
model_name='submission', model_name="submission",
name='notes', name="notes",
field=models.TextField(blank=True, help_text='Optional notes about the submission', null=True), field=models.TextField(
blank=True, help_text="Optional notes about the submission", null=True
),
), ),
] ]
opus_submitter/submissions/migrations/0006_remove_puzzleresponse_ocr_confidence_score_and_more.py
+24 -13
View File
@@ -4,29 +4,40 @@ from django.db import migrations, models
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('submissions', '0005_alter_submission_notes'), ("submissions", "0005_alter_submission_notes"),
] ]
operations = [ operations = [
migrations.RemoveField( migrations.RemoveField(
model_name='puzzleresponse', model_name="puzzleresponse",
name='ocr_confidence_score', name="ocr_confidence_score",
), ),
migrations.AddField( migrations.AddField(
model_name='puzzleresponse', model_name="puzzleresponse",
name='ocr_confidence_area', name="ocr_confidence_area",
field=models.FloatField(blank=True, help_text='OCR confidence score for area (0.0 to 1.0)', null=True), field=models.FloatField(
blank=True,
help_text="OCR confidence score for area (0.0 to 1.0)",
null=True,
),
), ),
migrations.AddField( migrations.AddField(
model_name='puzzleresponse', model_name="puzzleresponse",
name='ocr_confidence_cost', name="ocr_confidence_cost",
field=models.FloatField(blank=True, help_text='OCR confidence score for cost (0.0 to 1.0)', null=True), field=models.FloatField(
blank=True,
help_text="OCR confidence score for cost (0.0 to 1.0)",
null=True,
),
), ),
migrations.AddField( migrations.AddField(
model_name='puzzleresponse', model_name="puzzleresponse",
name='ocr_confidence_cycles', name="ocr_confidence_cycles",
field=models.FloatField(blank=True, help_text='OCR confidence score for cycles (0.0 to 1.0)', null=True), field=models.FloatField(
blank=True,
help_text="OCR confidence score for cycles (0.0 to 1.0)",
null=True,
),
), ),
] ]
opus_submitter/submissions/migrations/0007_submission_manual_validation_requested.py
+7 -5
View File
@@ -4,15 +4,17 @@ from django.db import migrations, models
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('submissions', '0006_remove_puzzleresponse_ocr_confidence_score_and_more'), ("submissions", "0006_remove_puzzleresponse_ocr_confidence_score_and_more"),
] ]
operations = [ operations = [
migrations.AddField( migrations.AddField(
model_name='submission', model_name="submission",
name='manual_validation_requested', name="manual_validation_requested",
field=models.BooleanField(default=False, help_text='Whether the user specifically requested manual validation'), field=models.BooleanField(
default=False,
help_text="Whether the user specifically requested manual validation",
),
), ),
] ]
opus_submitter/submissions/models.py
+3 -5
View File
@@ -1,9 +1,7 @@
from django.db import models from django.db import models
from django.contrib.auth import get_user_model from django.contrib.auth import get_user_model
from django.utils import timezone
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
import uuid import uuid
import os
User = get_user_model() User = get_user_model()
@@ -242,11 +240,11 @@ class Submission(models.Model):
validated_at = models.DateTimeField( validated_at = models.DateTimeField(
null=True, blank=True, help_text="When this submission was validated" null=True, blank=True, help_text="When this submission was validated"
) )
# Manual validation request # Manual validation request
manual_validation_requested = models.BooleanField( manual_validation_requested = models.BooleanField(
default=False, default=False,
help_text="Whether the user specifically requested manual validation" help_text="Whether the user specifically requested manual validation",
) )
# Timestamps # Timestamps
opus_submitter/submissions/schemas.py
+1 -2
View File
@@ -1,5 +1,4 @@
from ninja import Schema, ModelSchema, File from ninja import Schema, ModelSchema
from ninja.files import UploadedFile
from typing import List, Optional from typing import List, Optional
from datetime import datetime from datetime import datetime
from uuid import UUID from uuid import UUID
opus_submitter/submissions/tests.py
-2
View File
@@ -1,3 +1 @@
from django.test import TestCase
# Create your tests here. # Create your tests here.
opus_submitter/submissions/utils.py
+45 -33
View File
@@ -4,6 +4,7 @@ Utilities for fetching Steam Workshop collection data using Steam Web API
import re import re
import requests import requests
from submissions.models import SteamCollection
from datetime import datetime from datetime import datetime
from django.utils import timezone from django.utils import timezone
from django.conf import settings from django.conf import settings
@@ -20,22 +21,28 @@ class SteamAPIClient:
def __init__(self, api_key: Optional[str] = None): def __init__(self, api_key: Optional[str] = None):
# Priority: parameter > database > settings > environment # Priority: parameter > database > settings > environment
self.api_key = api_key or self._get_api_key_from_db() or getattr(settings, "STEAM_API_KEY", None) self.api_key = (
api_key
or self._get_api_key_from_db()
or getattr(settings, "STEAM_API_KEY", None)
)
self.session = requests.Session() self.session = requests.Session()
if not self.api_key: if not self.api_key:
logger.warning("No Steam API key provided. Some features may be limited.") logger.warning("No Steam API key provided. Some features may be limited.")
def _get_api_key_from_db(self) -> Optional[str]: def _get_api_key_from_db(self) -> Optional[str]:
"""Get active API key from database""" """Get active API key from database"""
try: try:
from .models import SteamAPIKey from .models import SteamAPIKey
api_key_obj = SteamAPIKey.get_active_key() api_key_obj = SteamAPIKey.get_active_key()
if api_key_obj: if api_key_obj:
# Update last_used timestamp # Update last_used timestamp
from django.utils import timezone from django.utils import timezone
api_key_obj.last_used = timezone.now() api_key_obj.last_used = timezone.now()
api_key_obj.save(update_fields=['last_used']) api_key_obj.save(update_fields=["last_used"])
return api_key_obj.api_key return api_key_obj.api_key
except Exception as e: except Exception as e:
logger.debug(f"Could not fetch API key from database: {e}") logger.debug(f"Could not fetch API key from database: {e}")
@@ -234,55 +241,59 @@ class SteamCollectionFetcher:
def _fetch_collection_items_via_api(self, collection_id: str) -> List[Dict]: def _fetch_collection_items_via_api(self, collection_id: str) -> List[Dict]:
""" """
Fetch collection items using GetCollectionDetails API Fetch collection items using GetCollectionDetails API
Args: Args:
collection_id: Steam collection ID collection_id: Steam collection ID
Returns: Returns:
List of item dictionaries List of item dictionaries
""" """
items = [] items = []
try: try:
# Use GetCollectionDetails API to get collection items # Use GetCollectionDetails API to get collection items
url = f"{self.api_client.BASE_URL}/ISteamRemoteStorage/GetCollectionDetails/v1/" url = f"{self.api_client.BASE_URL}/ISteamRemoteStorage/GetCollectionDetails/v1/"
data = { data = {"collectioncount": 1, "publishedfileids[0]": collection_id}
'collectioncount': 1,
'publishedfileids[0]': collection_id
}
response = self.api_client.session.post(url, data=data, timeout=30) response = self.api_client.session.post(url, data=data, timeout=30)
if response.status_code == 200: if response.status_code == 200:
collection_response = response.json() collection_response = response.json()
if 'response' in collection_response and 'collectiondetails' in collection_response['response']: if (
for collection in collection_response['response']['collectiondetails']: "response" in collection_response
if collection.get('result') == 1 and 'children' in collection: and "collectiondetails" in collection_response["response"]
):
for collection in collection_response["response"][
"collectiondetails"
]:
if collection.get("result") == 1 and "children" in collection:
# Extract item IDs with their sort order # Extract item IDs with their sort order
child_items = [] child_items = []
for child in collection['children']: for child in collection["children"]:
if 'publishedfileid' in child: if "publishedfileid" in child:
child_items.append({ child_items.append(
'id': str(child['publishedfileid']), {
'sort_order': child.get('sortorder', 0) "id": str(child["publishedfileid"]),
}) "sort_order": child.get("sortorder", 0),
}
)
# Sort by sort order to maintain collection order # Sort by sort order to maintain collection order
child_items.sort(key=lambda x: x['sort_order']) child_items.sort(key=lambda x: x["sort_order"])
item_ids = [item['id'] for item in child_items] item_ids = [item["id"] for item in child_items]
if item_ids: if item_ids:
items = self._fetch_items_by_ids(item_ids) items = self._fetch_items_by_ids(item_ids)
except Exception as e: except Exception as e:
logger.error(f"Failed to fetch collection items via API: {e}") logger.error(f"Failed to fetch collection items via API: {e}")
return items return items
def _fetch_items_by_ids(self, item_ids: List[str]) -> List[Dict]: def _fetch_items_by_ids(self, item_ids: List[str]) -> List[Dict]:
"""Fetch item details by their IDs""" """Fetch item details by their IDs"""
items = [] items = []
# Fetch details for all items in batches (Steam API has limits) # Fetch details for all items in batches (Steam API has limits)
batch_size = 20 # Conservative batch size batch_size = 20 # Conservative batch size
for i in range(0, len(item_ids), batch_size): for i in range(0, len(item_ids), batch_size):
@@ -300,7 +311,7 @@ class SteamCollectionFetcher:
): ):
item_id = item_data.get("publishedfileid", "unknown") item_id = item_data.get("publishedfileid", "unknown")
result = item_data.get("result", 0) result = item_data.get("result", 0)
if result == 1: # Success if result == 1: # Success
item_info = { item_info = {
"steam_item_id": str(item_id), "steam_item_id": str(item_id),
@@ -333,14 +344,15 @@ class SteamCollectionFetcher:
items.append(item_info) items.append(item_info)
else: else:
# Log failed items # Log failed items
logger.warning(f"Failed to fetch item {item_id}: result={result}, ban_reason={item_data.get('ban_reason', 'N/A')}") logger.warning(
f"Failed to fetch item {item_id}: result={result}, ban_reason={item_data.get('ban_reason', 'N/A')}"
)
except Exception as e: except Exception as e:
logger.error(f"Failed to fetch batch of collection items: {e}") logger.error(f"Failed to fetch batch of collection items: {e}")
continue continue
return items return items
def fetch_steam_collection(url: str) -> Dict: def fetch_steam_collection(url: str) -> Dict:
@@ -357,7 +369,7 @@ def fetch_steam_collection(url: str) -> Dict:
return fetcher.fetch_collection_data(url) return fetcher.fetch_collection_data(url)
def create_or_update_collection(url: str) -> Tuple["SteamCollection", bool]: def create_or_update_collection(url: str) -> Tuple[SteamCollection, bool]:
""" """
Create or update a Steam collection in the database Create or update a Steam collection in the database
opus_submitter/submissions/views.py
-2
View File
@@ -1,3 +1 @@
from django.shortcuts import render
# Create your views here. # Create your views here.