|
| 1 | +"""Files uploaded from the user's computer into a project's Uploads folder. |
| 2 | +
|
| 3 | +A volume, a mask or a reference made on one's own machine has to reach the |
| 4 | +server before a protocol can use it. An upload goes in chunks, so a big file |
| 5 | +neither fills the server's memory nor passes a proxy's request limit, and a |
| 6 | +dropped connection (a VPN) resumes where it stopped: |
| 7 | +
|
| 8 | +1. ``startUpload`` checks the name, the size limit (``SCIPIONWEB_UPLOAD_MAX_GB``) |
| 9 | + and the free disk space, and opens a partial file; |
| 10 | +2. ``writeChunk`` appends each chunk at the offset the client says, which must |
| 11 | + be what was received so far (``uploadStatus`` tells it after a drop); |
| 12 | +3. ``completeUpload`` gives the file its name, never over an existing file. |
| 13 | +
|
| 14 | +Partial files and what they are (``<id>.json``) live in ``Uploads/.partial``; |
| 15 | +nothing is kept in memory, so an upload survives a restart of the server. |
| 16 | +Partial files untouched for a day are removed when a new upload starts. |
| 17 | +""" |
| 18 | +from __future__ import annotations |
| 19 | + |
| 20 | +import json |
| 21 | +import os |
| 22 | +import shutil |
| 23 | +import time |
| 24 | +import uuid |
| 25 | +from dataclasses import dataclass |
| 26 | +from pathlib import Path |
| 27 | +from typing import Optional |
| 28 | + |
| 29 | +UPLOADS_FOLDER = "Uploads" |
| 30 | +PARTIAL_FOLDER = ".partial" |
| 31 | +DEFAULT_MAX_GB = 10.0 |
| 32 | +# The largest chunk one request may carry; the web sends 8 MB |
| 33 | +MAX_CHUNK_BYTES = 64 * 1024 * 1024 |
| 34 | +# Free space left on the disk after an upload, so a full disk does not stop the database |
| 35 | +FREE_SPACE_MARGIN_BYTES = 2 * 1024 ** 3 |
| 36 | +STALE_PARTIAL_SECONDS = 24 * 3600 |
| 37 | +MAX_NAME_LENGTH = 255 |
| 38 | + |
| 39 | + |
| 40 | +class UploadError(Exception): |
| 41 | + """A refused upload: status is the HTTP status the route answers.""" |
| 42 | + |
| 43 | + def __init__(self, status: int, message: str, received: Optional[int] = None): |
| 44 | + super().__init__(message) |
| 45 | + self.status = status |
| 46 | + self.message = message |
| 47 | + self.received = received |
| 48 | + |
| 49 | + |
| 50 | +@dataclass(frozen=True) |
| 51 | +class Upload: |
| 52 | + uploadId: str |
| 53 | + name: str |
| 54 | + size: int |
| 55 | + received: int |
| 56 | + userId: int |
| 57 | + |
| 58 | + def toDict(self) -> dict: |
| 59 | + return {"uploadId": self.uploadId, "name": self.name, "size": self.size, "received": self.received, |
| 60 | + "path": "%s/%s" % (UPLOADS_FOLDER, self.name)} |
| 61 | + |
| 62 | + |
| 63 | +def maxUploadBytes() -> int: |
| 64 | + """The size limit of one file (SCIPIONWEB_UPLOAD_MAX_GB; 0 or less: no limit).""" |
| 65 | + try: |
| 66 | + gigabytes = float(os.environ.get("SCIPIONWEB_UPLOAD_MAX_GB") or DEFAULT_MAX_GB) |
| 67 | + except ValueError: |
| 68 | + gigabytes = DEFAULT_MAX_GB |
| 69 | + return int(gigabytes * 1024 ** 3) if gigabytes > 0 else 0 |
| 70 | + |
| 71 | + |
| 72 | +def safeName(name: str) -> str: |
| 73 | + """The file name an upload is kept under: a plain, visible name, never a path.""" |
| 74 | + text = str(name or "").strip() |
| 75 | + if (not text or text in (".", "..") or text.startswith(".") or len(text) > MAX_NAME_LENGTH |
| 76 | + or any(char in text for char in ("/", "\\", "\0")) or any(ord(char) < 32 for char in text)): |
| 77 | + raise UploadError(400, "Invalid file name: %r" % str(name or "")) |
| 78 | + return text |
| 79 | + |
| 80 | + |
| 81 | +def uploadsFolder(projectRoot: Path) -> Path: |
| 82 | + return Path(projectRoot) / UPLOADS_FOLDER |
| 83 | + |
| 84 | + |
| 85 | +def _partialFolder(projectRoot: Path) -> Path: |
| 86 | + return uploadsFolder(projectRoot) / PARTIAL_FOLDER |
| 87 | + |
| 88 | + |
| 89 | +def _validId(uploadId: str) -> str: |
| 90 | + text = str(uploadId or "") |
| 91 | + if len(text) != 32 or any(char not in "0123456789abcdef" for char in text): |
| 92 | + raise UploadError(404, "Upload not found") |
| 93 | + return text |
| 94 | + |
| 95 | + |
| 96 | +def _paths(projectRoot: Path, uploadId: str): |
| 97 | + folder = _partialFolder(projectRoot) |
| 98 | + uploadId = _validId(uploadId) |
| 99 | + return folder / (uploadId + ".part"), folder / (uploadId + ".json") |
| 100 | + |
| 101 | + |
| 102 | +def _load(projectRoot: Path, uploadId: str, userId: int) -> Upload: |
| 103 | + data, meta = _paths(projectRoot, uploadId) |
| 104 | + try: |
| 105 | + info = json.loads(meta.read_text(encoding="utf-8")) |
| 106 | + received = data.stat().st_size |
| 107 | + except (OSError, ValueError): |
| 108 | + raise UploadError(404, "Upload not found") |
| 109 | + # Only who started an upload may go on with it |
| 110 | + if int(info.get("userId", -1)) != int(userId): |
| 111 | + raise UploadError(404, "Upload not found") |
| 112 | + return Upload(uploadId, str(info["name"]), int(info["size"]), received, int(userId)) |
| 113 | + |
| 114 | + |
| 115 | +def _removeStalePartials(folder: Path, now: float) -> None: |
| 116 | + try: |
| 117 | + entries = list(folder.iterdir()) |
| 118 | + except OSError: |
| 119 | + return |
| 120 | + for entry in entries: |
| 121 | + try: |
| 122 | + if entry.is_file() and now - entry.stat().st_mtime > STALE_PARTIAL_SECONDS: |
| 123 | + entry.unlink() |
| 124 | + except OSError: |
| 125 | + continue |
| 126 | + |
| 127 | + |
| 128 | +def startUpload(projectRoot: Path, userId: int, name: str, size: int) -> Upload: |
| 129 | + name = safeName(name) |
| 130 | + if not isinstance(size, int) or size < 0: |
| 131 | + raise UploadError(400, "Invalid file size") |
| 132 | + limit = maxUploadBytes() |
| 133 | + if limit and size > limit: |
| 134 | + raise UploadError(413, "%s is %.1f GB; files up to %.1f GB can be uploaded" |
| 135 | + % (name, size / 1024 ** 3, limit / 1024 ** 3)) |
| 136 | + target = uploadsFolder(projectRoot) / name |
| 137 | + if target.exists() or target.is_symlink(): |
| 138 | + raise UploadError(409, "Uploads/%s already exists" % name) |
| 139 | + folder = _partialFolder(projectRoot) |
| 140 | + folder.mkdir(parents=True, exist_ok=True) |
| 141 | + free = shutil.disk_usage(folder).free |
| 142 | + if size + FREE_SPACE_MARGIN_BYTES > free: |
| 143 | + raise UploadError(507, "Not enough free disk space on the server for %s (%.1f GB free)" |
| 144 | + % (name, free / 1024 ** 3)) |
| 145 | + _removeStalePartials(folder, time.time()) |
| 146 | + uploadId = uuid.uuid4().hex |
| 147 | + data, meta = _paths(projectRoot, uploadId) |
| 148 | + data.touch() |
| 149 | + meta.write_text(json.dumps({"name": name, "size": size, "userId": int(userId)}), encoding="utf-8") |
| 150 | + return Upload(uploadId, name, size, 0, int(userId)) |
| 151 | + |
| 152 | + |
| 153 | +def uploadStatus(projectRoot: Path, userId: int, uploadId: str) -> Upload: |
| 154 | + return _load(projectRoot, uploadId, userId) |
| 155 | + |
| 156 | + |
| 157 | +def writeChunk(projectRoot: Path, userId: int, uploadId: str, offset: int, chunk: bytes) -> Upload: |
| 158 | + """Append chunk at offset, which must be what was received so far (409 tells the right one).""" |
| 159 | + upload = _load(projectRoot, uploadId, userId) |
| 160 | + if offset != upload.received: |
| 161 | + raise UploadError(409, "Expected the chunk at byte %d" % upload.received, received=upload.received) |
| 162 | + if len(chunk) > MAX_CHUNK_BYTES: |
| 163 | + raise UploadError(413, "Chunks may be up to %d bytes" % MAX_CHUNK_BYTES) |
| 164 | + if upload.received + len(chunk) > upload.size: |
| 165 | + raise UploadError(400, "More data than the declared size of %d bytes" % upload.size) |
| 166 | + data, _ = _paths(projectRoot, uploadId) |
| 167 | + with open(data, "ab") as handle: |
| 168 | + handle.write(chunk) |
| 169 | + return Upload(upload.uploadId, upload.name, upload.size, upload.received + len(chunk), upload.userId) |
| 170 | + |
| 171 | + |
| 172 | +def completeUpload(projectRoot: Path, userId: int, uploadId: str) -> Upload: |
| 173 | + upload = _load(projectRoot, uploadId, userId) |
| 174 | + if upload.received != upload.size: |
| 175 | + raise UploadError(409, "Only %d of %d bytes were received" % (upload.received, upload.size), |
| 176 | + received=upload.received) |
| 177 | + data, meta = _paths(projectRoot, uploadId) |
| 178 | + target = uploadsFolder(projectRoot) / upload.name |
| 179 | + try: |
| 180 | + # A link fails if the name is taken meanwhile: an existing file is never replaced |
| 181 | + os.link(data, target) |
| 182 | + except FileExistsError: |
| 183 | + raise UploadError(409, "Uploads/%s already exists" % upload.name) |
| 184 | + data.unlink() |
| 185 | + meta.unlink(missing_ok=True) |
| 186 | + return upload |
| 187 | + |
| 188 | + |
| 189 | +def cancelUpload(projectRoot: Path, userId: int, uploadId: str) -> None: |
| 190 | + _load(projectRoot, uploadId, userId) |
| 191 | + for path in _paths(projectRoot, uploadId): |
| 192 | + path.unlink(missing_ok=True) |
0 commit comments