reorganize into a pip-installable package

This commit is contained in:
adam j hartz 2023-03-18 21:31:25 -04:00
commit a8fa2de6b8
Signed by: hz
GPG key ID: B63A0A59F5C82493
11 changed files with 100 additions and 4 deletions

17
.gitignore vendored Normal file
View file

@ -0,0 +1,17 @@
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg

17
README
View file

@ -1,6 +1,19 @@
gradesoop: grading of printed exams within catsoop
install prereqs:
install prereqs and the gradesoop package:
$ sudo apt install poppler-utils libzbar-dev
$ pip install -r requirements.txt
$ pip install .
this will install the gradesoop command, which is the entry point for all of
the pdf processing stuff. run the following to get started:
$ gradesoop --help
all of the subcommands have a --help flag that shows usage, arguments, and
options. here they are, roughly in the order they're needed:
* `gradesoop add-qrs` for adding qr codes to exams
* `gradesoop parse-scans` for reading and organizing raw scan data
* `gradesoop separate` for generating per-student pdfs from scan data
* `gradesoop setup-catsoop-course` for adding the web interface to a course

3
gradesoop/.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
*.pdf
*.pickle
__pycache__

1
gradesoop/__init__.py Normal file
View file

@ -0,0 +1 @@
__version__ = "0.0.0"

15
gradesoop/__main__.py Normal file
View file

@ -0,0 +1,15 @@
#!/usr/bin/env python3
import click
from .cli.add_qrs import main as add_qrs
from .cli.gather_scan_data import main as gather_scan_data
from .cli.separate_exams import main as separate_exams
main = click.Group()
main.add_command(add_qrs, "add-qrs")
main.add_command(gather_scan_data, "parse-scans")
main.add_command(separate_exams, "separate")
if __name__ == "__main__":
main()

145
gradesoop/cli/add_qrs.py Executable file
View file

@ -0,0 +1,145 @@
#!/usr/bin/env python3
import io
import os
import hashlib
import multiprocessing
import click
import pyqrcode
import matplotlib
import matplotlib.cm
import matplotlib.figure
from pypdf import PdfReader, PdfWriter
from matplotlib.backends.backend_pdf import FigureCanvasPdf as mpl_FigureCanvasPdf
BAD_NUMBERS = set(map(str, (14, 69, 88, 420)))
from ..geometry import PAGE_SIZE, QR_SIZE, QR_LOCS
@click.command()
@click.option("--start-num", "-s", default=0, type=int)
@click.option("--count", "-c", default=1, type=int)
@click.option("--output-dir", "-o", default=os.getcwd())
@click.option("--allow-bad-numbers", is_flag=True, default=False)
@click.option("--num-workers", "-j", default="0")
@click.argument("input_file")
def main(input_file, start_num, count, output_dir, allow_bad_numbers, num_workers):
assert count >= 1, "Need to produce at least one document"
padding = len(str(count))
assert padding <= 4
assert os.path.isfile(input_file), r"Not a file: {input_file}"
with open(input_file, "rb") as f:
exam_hash = str(int(hashlib.sha256(f.read()).hexdigest(), 16))[:5]
nums = []
current = start_num - 1
while len(nums) < count:
current += 1
serial_string = str(current).zfill(4)
if (not allow_bad_numbers) and any(i in serial_string for i in BAD_NUMBERS):
continue
nums.append(serial_string)
if num_workers == "all":
num_workers = multiprocessing.cpu_count()
num_workers = int(num_workers)
if num_workers == 0:
num_workers = multiprocessing.cpu_count() // 2 + 1
with multiprocessing.Pool(num_workers) as pool:
pool.map(add_qr_codes, [(input_file, i, exam_hash, output_dir) for i in nums])
print(
f"This exam's magic number is {exam_hash}. "
"Pass that to gather_scan_data.py if you want it to verify that the "
"scans came from this exam."
)
def add_qr_codes(inp):
input_file, serial_string, exam_hash, output_dir = inp
input_pdf_reader = PdfReader(input_file)
output_pdf_writer = PdfWriter()
for page_i, page in enumerate(input_pdf_reader.pages):
page = input_pdf_reader.pages[page_i]
fig = matplotlib.figure.Figure(figsize=PAGE_SIZE)
ax = fig.add_axes([0.0, 0.0, 1.0, 1.0])
ax.set_xlim(0, PAGE_SIZE[0])
ax.set_ylim(0, PAGE_SIZE[1])
ax.invert_yaxis()
for side in ("top", "bottom", "left", "right"):
ax.spines[side].set_visible(False)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
for qrnum, qr_loc in enumerate(QR_LOCS):
args = [
exam_hash,
f"{page_i+1:02d}",
str(qrnum),
serial_string,
]
qrcode = pyqrcode.create(
"".join(args), error="H", version=1, mode="numeric"
)
bmp = qrcode.code
center_x = qr_loc[0]
center_y = qr_loc[1]
left = center_x - QR_SIZE / 2
right = center_x + QR_SIZE / 2
bottom = center_y + QR_SIZE / 2
top = center_y - QR_SIZE / 2
ax.imshow(
bmp,
aspect=1,
interpolation="none",
cmap=matplotlib.cm.binary,
origin="upper",
extent=(left, right, bottom, top),
)
ax.text(
s=",".join(args[1:]),
x=center_x,
y=center_y + QR_SIZE / 2 + 0.1,
horizontalalignment="center",
verticalalignment="top",
fontsize=8,
)
canvas_pdf = mpl_FigureCanvasPdf(fig)
qrcode_fake_page = io.BytesIO()
for ax in fig.axes:
ax.patch.set_facecolor("none")
ax.patch.set_edgecolor("none")
fig.set_frameon(False)
canvas_pdf.print_pdf(qrcode_fake_page)
qrcode_fake_page.seek(0)
overlay_pdf_reader = PdfReader(qrcode_fake_page)
assert len(overlay_pdf_reader.pages) == 1
overlay_page = overlay_pdf_reader.pages[0]
page.merge_page(overlay_page)
output_pdf_writer.add_page(page)
os.makedirs(output_dir, exist_ok=True)
with open(os.path.join(output_dir, f"exam_{serial_string}.pdf"), "wb") as f:
output_pdf_writer.write(f)
print("Done with exam", serial_string)
if __name__ == "__main__":
main()

160
gradesoop/cli/gather_scan_data.py Executable file
View file

@ -0,0 +1,160 @@
#!/usr/bin/env python3
import io
import os
import sys
import pickle
import multiprocessing
import click
from PIL import Image
from pyzbar import pyzbar
from pypdf import PdfReader
def parse(code):
assert len(code) == 12
return list(map(int, [code[:5], code[5:7], code[7:8], code[8:12]]))
def get_qrs(x):
fname, reader, page_i = x
print("Scanning", fname, "page", page_i + 1)
buf = io.BytesIO()
buf.write(reader.pages[page_i].images[0].data)
im = Image.open(buf).convert("L")
qrinfo = pyzbar.decode(im)
if len(qrinfo) == 2:
return qrinfo, False
# check for upside-down (and maybe pre-fix it?)
y_coords = [p[1] for i in qrinfo for p in tuple(i.polygon)]
rotate = any(y < im.size[1] // 3 for y in y_coords)
if rotate:
# rotate the image and try again
im = im.rotate(180)
qrinfo = pyzbar.decode(im)
return qrinfo, rotate
@click.command()
@click.option("--verify")
@click.option("--append", "-a", is_flag=True, default=False)
@click.option("--min-pages", "-p", type=int)
@click.option("--output", "-o", required=True)
@click.option("--force", "-f", is_flag=True, default=False)
@click.option("--num-workers", "-j", default="0")
@click.argument("input_file", nargs=-1)
def main(input_file, verify, min_pages, append, output, force, num_workers):
if num_workers == "all":
num_workers = multiprocessing.cpu_count()
num_workers = int(num_workers)
if num_workers == 0:
num_workers = multiprocessing.cpu_count() // 2 + 1
pool = multiprocessing.Pool(num_workers)
if os.path.isfile(output) and (not append) and (not force):
print(
"Output already exists and --append not specified. Exiting. (To avoid this and blow away the file, add -f)."
)
sys.exit(1)
if append:
with open(output, "rb") as f:
seen_files, pages_map = pickle.load(f)
else:
pages_map = {}
seen_files = set()
# scan for qrs
for fname in input_file:
fname = os.path.realpath(fname)
if fname in seen_files:
continue
reader = PdfReader(fname)
qrinfos = pool.map(
get_qrs, [(fname, reader, i) for i in range(len(reader.pages))]
)
for page_i, (qrinfo, rotated) in enumerate(qrinfos):
if len(qrinfo) == 0:
continue # bad scan, skip it
qrs = {}
for info in qrinfo:
magic, page, qrnum, serial = parse(info.data)
qrs[qrnum] = (
serial,
page,
magic,
tuple(map(tuple, info.polygon)),
)
serials, pagenums, magics, _ = list(map(set, zip(*qrs.values())))
if verify is not None and magics != {verify}:
continue
if {len(serials), len(pagenums)} != {1}:
continue
old = pages_map.get(serial, {}).get(page, None)
if old is not None and (len(old[-1]) == 2 or len(old[-1]) > len(qrs)):
continue
if len(qrs) == 2 and set(qrs) != {0, 1}:
continue
pages_map.setdefault(serial, {})[page] = (
fname,
page_i,
rotated,
qrs,
)
seen_files.add(fname)
if min_pages is None:
min_pages = max(max(i) for i in pages_map.values())
expected_pages = set(range(1, min_pages + 1))
# sanity check results
print(f"{len(pages_map)} exams encountered:")
print(f'{", ".join(map(str, sorted(pages_map)))}')
missing_pages = {}
single_pages = {}
for serial, pages in pages_map.items():
missing = expected_pages - set(pages)
if missing:
missing_pages[serial] = missing
ones = {page for page, qrs in pages.items() if len(qrs[-1]) == 1}
if ones:
single_pages[serial] = ones
print()
if missing_pages:
print(
"\x1B[1;31mEXAMS MISSING PAGES (YOU MAY WANT TO ADD ADDITIONAL SCANS BEFORE MOVING ON):\x1B[0m"
)
for serial, missing in missing_pages.items():
print(
f' * {serial}: missing page(s) {",".join(map(str, sorted(missing)))}'
)
else:
print("\x1B[1;32mAll exams have the required pages.\x1B[0m")
if single_pages:
print()
print(
"\x1B[1;31mSome pages only have one QR code (rectification data will be guessed):\x1B[0m"
)
for serial, single in sorted(single_pages.items()):
print(f' * exam {serial}: page(s) {",".join(map(str, sorted(single)))}')
else:
print("\x1B[1;32mAll pages have two QR codes.\x1B[0m")
with open(output, "wb") as f:
pickle.dump((seen_files, pages_map), f)
if __name__ == "__main__":
main()

101
gradesoop/cli/separate_exams.py Executable file
View file

@ -0,0 +1,101 @@
#!/usr/bin/env python3
import io
import os
import numpy
import pickle
import tqdm
import click
import img2pdf
from PIL import Image, ImageTransform
from pypdf import PdfReader
from ..geometry import (
PAGE_SIZE,
QR_EXPECTED_POINTS,
find_transform_matrix,
)
@click.command()
@click.option("--rectify/--no-rectify", default=True)
@click.option("--output-dir", "-o", required=True)
@click.option("--exam-numbers", "-e")
@click.option("--output-dpi", type=int, default=200)
@click.option("--fill-color", type=int, default=200)
@click.argument("input_file")
def main(input_file, rectify, output_dir, output_dpi, fill_color, exam_numbers):
os.makedirs(output_dir, exist_ok=True)
with open(input_file, "rb") as f:
pdf_fnames, exam_data = pickle.load(f)
reader_map = {}
for fname in pdf_fnames:
reader_map[fname] = PdfReader(fname)
exams_to_do = set(exam_data)
if exam_numbers is not None:
exams_to_do = {int(i) for i in exam_numbers.split(",")}
for exam_number, exam_info in sorted(exam_data.items()):
if exam_number not in exams_to_do:
continue
page_images = {}
for page, (filename, page_number, rotated, qr_codes) in exam_info.items():
im_buf = io.BytesIO()
im_buf.write(reader_map[filename].pages[page_number].images[0].data)
image = Image.open(im_buf)
if rotated:
image = image.rotate(180)
output_size = tuple(int(output_dpi * i) for i in PAGE_SIZE)
if rectify:
points_measured = []
points_expected = []
for qr_ix in qr_codes:
points_expected.extend(QR_EXPECTED_POINTS[qr_ix])
points_measured.extend(qr_codes[qr_ix][-1])
points_measured = numpy.array(points_measured)
points_expected = numpy.array(points_expected) * output_dpi
transform_coeffs = sum(
find_transform_matrix(
points_expected, points_measured, True
).tolist(),
[],
)[:6]
transform = ImageTransform.AffineTransform(transform_coeffs)
image = transform.transform(
output_size,
image,
fillcolor=(
fill_color
if image.mode == "L"
else (fill_color, fill_color, fill_color)
),
)
print(output_size, image.size, rotated)
page_images[page] = io.BytesIO()
image.save(page_images[page], "JPEG")
page_images[page].seek(0)
new_fname = os.path.join(output_dir, f"{exam_number:04d}.pdf")
with open(new_fname, "wb") as f:
f.write(
img2pdf.convert(
[page for ix, page in sorted(page_images.items())],
pagesize="letter",
imgsize="letter",
)
)
print("wrote", new_fname)
if __name__ == "__main__":
main()

91
gradesoop/geometry.py Normal file
View file

@ -0,0 +1,91 @@
import numpy
PAGE_SIZE = (8.5, 11)
QR_SIZE = 0.5 # inches
QR_LOCS = [
(1.0, 11 - 0.75),
(8.5 - 0.75, 11 - 0.75),
]
# QR_OFFSETS maps us from zbar's coordinate frame (y increases upwards) to
# pillow's (y increases downwards)
QR_OFFSETS = QR_SIZE / 2 * numpy.array([[-1, -1], [-1, +1], [+1, +1], [+1, -1]])
QR_EXPECTED_POINTS = [(center + QR_OFFSETS).tolist() for center in QR_LOCS]
# copied from scikit-image:
# https://github.com/scikit-image/scikit-image/blob/fe547a30cac21859acce67481311d4edd0106056/skimage/transform/_geometric.py
# (BSD 3-clause from what i can tell)
def find_transform_matrix(src, dst, estimate_scale):
"""Estimate N-D similarity transformation with or without scaling.
Parameters
----------
src : (M, N) array
Source coordinates.
dst : (M, N) array
Destination coordinates.
estimate_scale : bool
Whether to estimate scaling factor.
Returns
-------
T : (N + 1, N + 1)
The homogeneous similarity transformation matrix. The matrix contains
NaN values only if the problem is not well-conditioned.
References
----------
.. [1] "Least-squares estimation of transformation parameters between two
point patterns", Shinji Umeyama, PAMI 1991, :DOI:`10.1109/34.88573`
"""
num = src.shape[0]
dim = src.shape[1]
# Compute mean of src and dst.
src_mean = src.mean(axis=0)
dst_mean = dst.mean(axis=0)
# Subtract mean from src and dst.
src_demean = src - src_mean
dst_demean = dst - dst_mean
# Eq. (38).
A = dst_demean.T @ src_demean / num
# Eq. (39).
d = numpy.ones((dim,), dtype=numpy.double)
if numpy.linalg.det(A) < 0:
d[dim - 1] = -1
T = numpy.eye(dim + 1, dtype=numpy.double)
U, S, V = numpy.linalg.svd(A)
# Eq. (40) and (43).
rank = numpy.linalg.matrix_rank(A)
if rank == 0:
return numpy.nan * T
if rank == dim - 1:
if numpy.linalg.det(U) * numpy.linalg.det(V) > 0:
T[:dim, :dim] = U @ V
else:
s = d[dim - 1]
d[dim - 1] = -1
T[:dim, :dim] = U @ numpy.diag(d) @ V
d[dim - 1] = s
else:
T[:dim, :dim] = U @ numpy.diag(d) @ V
if estimate_scale:
# Eq. (41) and (42).
scale = 1.0 / src_demean.var(axis=0).sum() * (S @ d)
else:
scale = 1.0
T[:dim, dim] = dst_mean - scale * (T[:dim, :dim] @ src_mean.T)
T[:dim, :dim] *= scale
return T

View file

@ -8,3 +8,4 @@ black
pillow
pyzbar
img2pdf
catsoop

43
setup.py Normal file
View file

@ -0,0 +1,43 @@
from setuptools import setup
from gradesoop import __version__ as VERSION
def main():
with open(os.path.join(os.path.dirname(__file__), "requirements.txt"), "r") as f:
requirements = f.read().split("\n")
with open(os.path.join(os.path.dirname(__file__), "README"), "r") as f:
readme = f.read()
setup(
name="gradesoop",
version=VERSION,
author="adam j hartz",
author_email="hz@mit.edu",
packages=["gradesoop", "gradesoop.cli"],
package_dir={"gradesoop": "gradesoop"},
scripts=[],
url="https://catsoop.org/git/catsoop/gradesoop",
license="AGPLv3+",
description="gradesoop: grading of printed exams within catsoop",
long_description=readme,
long_description_content_type="text/plain",
include_package_data=True,
entry_points={"console_scripts": ["gradesoop = gradesoop.__main__:main"]},
install_requires=requirements,
classifiers=[
"Development Status :: 2 - Pre-Alpha",
"License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
"Environment :: Console",
"Environment :: Web Environment",
"Operating System :: POSIX :: Linux",
"Programming Language :: Python :: 3 :: Only",
"Intended Audience :: Education",
"Topic :: Education",
],
)
if __name__ == "__main__":
main()