- 001
- 002
- 003
- 004
- 005
- 006
- 007
- 008
- 009
- 010
- 011
- 012
- 013
- 014
- 015
- 016
- 017
- 018
- 019
- 020
- 021
- 022
- 023
- 024
- 025
- 026
- 027
- 028
- 029
- 030
- 031
- 032
- 033
- 034
- 035
- 036
- 037
- 038
- 039
- 040
- 041
- 042
- 043
- 044
- 045
- 046
- 047
- 048
- 049
- 050
- 051
- 052
- 053
- 054
- 055
- 056
- 057
- 058
- 059
- 060
- 061
- 062
- 063
- 064
- 065
- 066
- 067
- 068
- 069
- 070
- 071
- 072
- 073
- 074
- 075
- 076
- 077
- 078
- 079
- 080
- 081
- 082
- 083
- 084
- 085
- 086
- 087
- 088
- 089
- 090
- 091
- 092
- 093
- 094
- 095
- 096
- 097
- 098
- 099
- 100
for course_id in COURSES: #конфиги для слабаков, реальные пацаны хардкодят константы
participants = {}
print(f'Downloading course {course_id}...')
assignments = M.get_modules(course_id, 'assign')
captions = {item['instance'] : item['name'] for item in assignments}
rawsubs_per_assignment = M.get_submissions(captions.keys())
files = []
print('\tParsing assignments... ',end='')
for i,assign in ProgressBar.each(rawsubs_per_assignment, length=50, donefmt='[OK]\n', errorfmt='[FAIL]\n'):
#самописный прогресс-бар. В скрипте, который 90% времени запускается как демон.
assign_id = assign['assignmentid']
if assign_id not in participants:
lst = M.list_participants(assign_id)
participants[assign_id] = {i['id'] : i['fullname'] for i in lst}
for rs in assign['submissions']:
subfiles = M.get_submission_files(rs)
for sf in subfiles:
f = StoredFile( #самописный псевдо-ORM
id = None,
url = sf['fileurl'],
filename = sf['filename'],
downloaded = False,
size = sf['filesize'],
mimetype = sf['mimetype'],
timestamp = sf['timemodified'],
course = course_id,
instance = assign_id,
submission = rs['id'],
userid = rs['userid'],
username = participants[assign_id][rs['userid']])
files.append(f)
print('\tTotal of {0} assignments and {1} files.'.format(len(participants), len(files)))
print('\tUpdating database... ', end='')
with DB:
for i,f in ProgressBar.each(files, captionfunc = (lambda f:f.filename),
length=25, valuefmt='{ratio:.0%} ', donefmt='[OK]\n', errorfmt='[FAIL]\n'):
DB.update_file(f) #execute_many()? зачем? И так сойдёт.
files_to_download = list(DB.get_nonready_files())
if not files_to_download:
print('All files are up to date.')
else:
ok_count = 0
error_count = 0
print('Downloading missing files... ',end='')
for i,f in ProgressBar.each(files_to_download,
captionfunc = (lambda f: f'[{error_count} errors] {f.filename}'),
length=25, valuefmt='{ratio:4.0%} ', donefmt='[OK]\n', errorfmt='[FAIL]\n'):
#исчо один прогрессбар
path = localpath(f)
dir = os.path.dirname(path)
os.makedirs(dir, exist_ok = True)
if not verify(f):
try:
with open(path, "wb") as dest:
#зачем проверять размер скачиваемого файла или наличие контента?
source = M.get_download_fileobj(f.url)
shutil.copyfileobj(source, dest)
DB.mark_downloaded(f)
except:
error_count += 1
else:
ok_count += 1
else:
ok_count += 1
if error_count > 0:
print(f'{error_count} files failed to download!')
comparisons = list(DB.get_files_to_match())
if comparisons:
print('Matching non-compared files... ',end='')
log = open('errors.txt', 'wt', encoding='utf-8') #logging для слабаков
for i,(f1,f2) in ProgressBar.each(comparisons, length=25, valuefmt='{pos}/{max}', donefmt='[OK]\n', errorfmt='[FAIL]\n'):
#оптимизировать цикл, чтобы не читать один и тот же файл по 50 раз? Нафиг, работает же.
path1, path2 = localpath(f1), localpath(f2)
ok = True
if not verify(f1):
ok = False
DB.mark_downloaded(f1, False)
if not verify(f2):
ok = False
DB.mark_downloaded(f2, False)
if ok:
content1, content2 = None, None
try:
content1 = extract_content(path1)
content2 = extract_content(path2)
similarity = compare_content(content1, content2)
except Exception as E:
if content1 is None:
log.write(path1+'\n')
elif content2 is None:
log.write(path1+'\n')
else:
log.write('Comparison error!\n')
log.write(str(E)+'\n')
else:
DB.save_diff(f1, f2, similarity)
finally: #пусть сборщик мусора порадуется
del content1
del content2
log.close()
Написал тут скрипт, самому стыдно.
Но работает.
Follow us!