прикреплённый файл:contest.py к странице FrBrGeorge/PythonCopypasteProof

   1 #!/usr/bin/env python3
   2 import sys
   3 
   4 import re
   5 import time
   6 import urllib.request
   7 import os
   8 import pickle
   9 from collections import defaultdict as table
  10 
  11 # ~/Загрузки/contest_113_20181218003430.tgz
  12 # argv: ~/Загрузки/contest_113_20181218003430.tgz
  13 
  14 # Run selection
  15 #	Download all runs
  16 #	Download selected runs (20)
  17 # X	Download OK runs
  18 #	Download OK and PR runs
  19 #	Download OK, PR, RJ, IG, PD, DQ runs
  20 #File name pattern
  21 #	Use Contest Id
  22 # X	Use run number
  23 #	Use user Id
  24 # X	Use user Login
  25 #	Use user Name
  26 # X	Use problem short name
  27 #	Use programming language short name
  28 # X	Use submit time
  29 # X	Use source language or content type suffix
  30 
  31 # No directory structure
  32 
  33 def mkt(st):
  34     return time.mktime(time.strptime(st,"%Y%m%d%H%M%S"))
  35 
  36 def table_list(): return table(list)
  37 def table_dict(): return table(dict)
  38 def table_set(): return table(set)
  39 
  40 CF = sys.argv[1]
  41 for sfx in sys.argv[2:]:
  42     CFDELETE = "{}.{}.tmp".format(CF,sfx)
  43     if os.path.isfile(CFDELETE):
  44         os.unlink(CFDELETE)
  45 
  46 URL = "http://uneex.ru/LecturesCMC/PythonIntro2019" #?action=raw
  47 RAW = "?action=raw"
  48 reBase=re.compile(b".*\[\[(/\w+).*<<Date.*<<Date.(.*)T.*")
  49 reTask=re.compile(b"<<EJCMC.131, *(\w+)")
  50 
  51 CFWeb = CF+".Web.tmp"
  52 if os.path.isfile(CFWeb):
  53     with open(CFWeb,"rb") as f:
  54         ClDt = pickle.load(f)
  55         ClTs = pickle.load(f)
  56 else:
  57     BaseR = urllib.request.urlopen(URL+RAW)
  58     BaseT = [b" ".join(reBase.findall(s)[0]).decode("utf").strip() for s in BaseR if reBase.match(s)]
  59     ClDt = "\n".join(BaseT)
  60 
  61     ClTs=""
  62     for s in ClDt.split('\n'):
  63         p, d = s.split()
  64         #PageR = urllib.request.urlopen(URL+"/"+p+RAW)
  65         time.sleep(1)
  66         for t in reTask.findall(urllib.request.urlopen(URL+"/"+p+RAW).read()):
  67             ClTs += "\n"+p+" "+t.decode("utf")
  68     print(ClDt)
  69     print(ClTs)
  70     with open(CFWeb,"wb") as f:
  71         pickle.dump(ClDt,f)
  72         pickle.dump(ClTs,f)
  73 
  74 ClDt = {l:mkt(d[:4]+d[5:7]+d[8:10]+"000000") for l,d in (s.split() for s in ClDt.split("\n") if s)}
  75 TsCl = {t:l for l,t in (s.split() for s in ClTs.split('\n') if s)}
  76 TsDt = {t:ClDt[l] for t,l in TsCl.items()}
  77 
  78 CFRes = CF+".All.tmp"
  79 if os.path.isfile(CFRes):
  80     with open(CFRes,"rb") as f:
  81         Res = pickle.load(f)
  82 else:
  83     import tarfile
  84     Res = table(set)
  85     with tarfile.open(CF,"r") as f:
  86         for o in f:
  87             if o.isfile():
  88                 s = o.name
  89                 # contest_86_20171219131616/000064-Nikscorp-HelloWorld-20170930113002.py
  90                 ID, *Nick, Task, Date = s.split("/")[-1][:-3].split("-")
  91                 Nick = "-".join(Nick)
  92                 Res[Nick].add(Task)
  93     with open(CFRes,"wb") as f:
  94         pickle.dump(Res,f)
  95 Users = {N for N,V in Res.items() if 3*len(V)>2*len(TsDt)}
  96 print("# Users: {} total / {} allowed".format(len(Res), len(Users)))
  97 
  98 CFSrc = CF+".Source.tmp"
  99 if os.path.isfile(CFSrc):
 100     with open(CFSrc,"rb") as f:
 101         IDs = pickle.load(f)
 102         Src = pickle.load(f)
 103         Tab = pickle.load(f)
 104 else:
 105     import tarfile
 106     Src = {}
 107     IDs = {}
 108     with tarfile.open(CF,"r") as f:
 109         for o in f:
 110             if o.isfile():
 111                 s = o.name
 112                 # contest_86_20171219131616/000064-Nikscorp-HelloWorld-20170930113002.py
 113                 ID, *Nick, Task, Date = s.split("/")[-1][:-3].split("-")
 114                 Nick, ID, Date = "-".join(Nick), int(ID), mkt(Date)
 115                 if Nick not in Users: continue
 116                 with f.extractfile(o) as df:
 117                     btxt = df.read()
 118                     try:
 119                         txt = btxt.decode()
 120                     except UnicodeDecodeError as E:
 121                         txt = btxt.decode("WINDOWS-1251")
 122                 Src[ID], IDs[ID] = txt, (Nick, Task, Date)
 123     Tab = table(table_set)
 124     for ID, (Nick, Task, Date) in IDs.items():
 125         Tab[Task][Nick].add(ID)
 126 
 127     with open(CFSrc,"wb") as f:
 128         pickle.dump(IDs,f)
 129         pickle.dump(Src,f)
 130         pickle.dump(Tab,f)
 131 print("# Tasks: {}/{}".format(len(IDs),max(IDs)))
 132 
 133 def mktask(ID):
 134     src = Src[ID]
 135     prep = rb.sub(" ",ra.sub("@",ast.dump(ast.parse(src,"ex.py"),annotate_fields=False)))
 136     txt = autopep8.fix_code(src)
 137     for i,p in enumerate(AstK):
 138         prep = prep.replace(p,chr(i+0x21)+" ")
 139     return txt, prep.replace(" ","")
 140 
 141 CFPrep = CF+".Prep.tmp"
 142 if os.path.isfile(CFPrep):
 143     with open(CFPrep,"rb") as f:
 144         Prep = pickle.load(f)
 145 else:
 146     import ast
 147     import re
 148     import autopep8
 149     import multiprocessing
 150     ra=re.compile(r"'[^']*'")
 151     rb=re.compile(r"[\[\]\{\}\(\)\,\ ]+")
 152     AstK = ['None ']+[s+' ' for s in dir(ast) if s[0].isalpha()]
 153     M = -1, int(max(IDs))
 154     pool = multiprocessing.Pool()
 155     S = sorted(Src)
 156     res = pool.map(mktask, S)
 157     #Prep = {}
 158     Prep = dict(zip(S, res))
 159     #for c, ID in enumerate(sorted(Src)):
 160     #    Prep[ID] = mktask(ID)
 161     #    if not c%100: print(c, end="\r")
 162     with open(CFPrep,"wb") as f:
 163         pickle.dump(Prep,f)
 164 print("# Total code/prepared: {}/{}".format(sum(len(t) for t,p in Prep.values()), sum(len(p) for t,p in Prep.values())))
 165 
 166 def getdist(id1, id2):
 167     return editdistance.eval(Prep[id1][1], Prep[id2][1])*2/(len(Prep[id1][1])+len(Prep[id2][1]))
 168 
 169 def cluster(Heap, D="@"):
 170     l=-1
 171     while l!=len(Heap):
 172         l = len(Heap)
 173         Heap={frozenset.union(*(b for b in Heap if a&b)) for a in Heap}
 174     return Heap
 175 
 176 def cluster2(I):
 177     Heap = { frozenset(c) for c in I.values() }
 178     return cluster(Heap)
 179 
 180 def calctask(T):
 181     print("*", T)
 182     P, R = set(), table(set)
 183     Us = sorted(Tab[T])
 184     for i in range(len(Us)-1):
 185         for j in range(i+1,len(Us)):
 186             for ID1 in Tab[T][Us[i]]:
 187                 R[ID1].add(ID1)
 188                 for ID2 in Tab[T][Us[j]]:
 189                     ID1, ID2 = sorted((ID1, ID2))
 190                     dist = getdist(ID1, ID2)
 191                     if dist<cPaste:
 192                         P.add((ID1, ID2, dist))
 193                         R[ID1].add(ID2)
 194     C = cluster2(R)
 195     #C = cluster({ frozenset({ i, j }) for i, j, d in P })
 196     return P, C
 197 
 198 CFPaste = CF+".Paste.tmp"
 199 cPaste = 0.01
 200 cRew = 0.1
 201 minCommon = 7
 202 
 203 if os.path.isfile(CFPaste):
 204     with open(CFPaste,"rb") as f:
 205         Paste = pickle.load(f)
 206 else:
 207     import editdistance
 208     import multiprocessing
 209     pool = multiprocessing.Pool()
 210     res = pool.map(calctask, Tab)
 211     Paste = dict(zip(Tab, res))
 212     with open(CFPaste,"wb") as f:
 213         pickle.dump(Paste,f)
 214 
 215 def tpasters(T):
 216     P = {}
 217     for C in Paste[T][1]:
 218         H, *L = sorted(C)
 219         U, L = IDs[H][0], {IDs[l][0] for l in L if IDs[l][0]!=IDs[H][0]}
 220         if not L: continue
 221         if len(L) >= minCommon:
 222             return None
 223         P[(U,H)] = L
 224     else:
 225         return P
 226 
 227 def ftpasters(T):
 228     P = {}
 229     for C in Paste[T][1]:
 230         H, *L = sorted(C)
 231         if {IDs[l][0] for l in L} == {IDs[H][0]}:
 232             continue
 233         if len(L) >= minCommon:
 234             return {}
 235         P[(IDs[H][0],H)] = [(IDs[l][0],l) for l in L if IDs[l][0]!=IDs[H][0]]
 236     return P
 237 
 238 def fpasters(T):
 239     P = {}
 240     for C in Paste[T][1]:
 241         H, *L = sorted(C)
 242         if {IDs[l][0] for l in L} == {IDs[H][0]}:
 243             continue
 244         P[(IDs[H][0],H)] = [(IDs[l][0],l) for l in L]
 245     return P
 246 
 247 print("# Pasters:")
 248 
 249 #{"HelloWorld","DummyClass", "NormalDouble","YieldFrom","SharedBrain","SectionShuffle","ParallelSegments","EvalFunction","CountInt","AndOr","DotsInCircle"}:
 250 
 251 import difflib
 252 import editdistance
 253 
 254 def shortest(T,U1,U2):
 255     I1 = {int(U1)} if U1.isdigit() else Tab[T][U1]
 256     I2 = {int(U2)} if U2.isdigit() else Tab[T][U2]
 257     I1,I2 = min(((i1,i2) for i1 in I1 for i2 in I2), key=lambda a: getdist(*a))
 258     U1, U2 = IDs[I1][0], IDs[I2][0]
 259     return (I1,U1),(I2,U2),getdist(I1,I2)
 260 
 261 def Allpasters():
 262     print(f"|||| '''{time.strftime('%F')}''' ||")
 263     for T in sorted(Paste):
 264         P = tpasters(T)
 265         print("|||| [[../Homework_{0}|{0}]]: {1}{2}||".format(T,sum(len(s) for s in Paste[T][1]),P and "." or "!"))
 266         if P and T not in TaskExs:
 267             for (U,H),L in sorted(P.items()):
 268                 print("||{} ({}): || {} ||".format(U,H," ".join(L)))
 269 
 270 def Taskpasters(T):
 271     for (U,H),L in fpasters(T).items():
 272         LL = " ".join("{}({})".format(*l) for l in L)
 273         print("{}({}): {}".format(U,H,LL))
 274 
 275 def Taskids(T,U=None):
 276     UU = [U] if U else Tab[T]
 277     print("#  {}: {}".format(T, " ".join("{}({})".format(u,i) for i,u in sorted((i,IDs[i][0]) for U in UU for i in Tab[T][U]))))
 278 
 279 def Userpastes(UU,TT=None):
 280     for T in ([TT] if TT else Paste):
 281         if not tpasters(T): continue
 282         for (U,H),L in fpasters(T).items():
 283             for u,i in L:
 284                 if UU == u or UU == U:
 285                     print("{} {}: {}({}) → {}({})".format(T,getdist(H,i),U,H,u,i))
 286 
 287 def Usertasks(U):
 288     print(" ".join(sorted(T for T in Tab if Tab[T][U])))
 289 
 290 def Diffsrc(T,U1,U2):
 291     (I1,U1),(I2,U2),d = shortest(T,U1,U2)
 292     print("{} ({}) / {} ({}): {}\n".format(U1,I1,U2,I2,d),"\n".join(difflib.ndiff(Src[I1].splitlines(),Src[I2].splitlines())))
 293 
 294 AuthIDs = set()
 295 TaskExs = {"ChainSlice"}
 296 def Mkpenalty(TaskEx=set(), addAUI=set()):
 297     Pen = {}
 298     AIDs = AuthIDs | addAUI
 299     for T in Tab:
 300         if T in TaskEx | TaskExs:
 301             Pen[T] = {}
 302             continue
 303         FP = ftpasters(T)
 304         #print(*(i for U,I in FP for u,i in FP[U,I] if u!=U))
 305         Pen[T]=set(i for U,I in FP for u,i in FP[U,I])
 306     return Pen
 307 Penalty = Mkpenalty()
 308 
 309 UP,MD,LW = 4,2,1
 310 Div =  (24*60*60,UP), (7*24*60*60,MD), (14*24*60*60,LW),
 311 def score(ID):
 312     U, T, D = IDs[ID]
 313     if ID in Penalty[T]: return 0
 314     for dd, sc in Div:
 315         if TsDt[T]+dd>=D:
 316             break
 317     return sc
 318 
 319 def Userscore(U):
 320     scores = [max(score(i) for i in Tab[T][U]) for T in Tab if Tab[T][U]]
 321     All = len(scores)
 322     Full = scores.count(4)
 323     return sum(scores), All, Full
 324 
 325 Mx = UP*len(Tab)
 326 Mn = Mx * 2 // 3
 327 def grade(M, g=("Отл","Хор","Хор","Удовл","Удовл")):
 328     if M<=Mn: return ""
 329     return g[int((Mx-M)*len(g)/(Mx-Mn))]
 330 
 331 def isID(i):
 332     if type(i) is int and i in IDs or type(i) is str and i.isdigit() and int(i) in IDs:
 333         return int(i)
 334     else:
 335         return None
 336 
 337 import os
 338 import readline
 339 import atexit
 340 
 341 C = []
 342 
 343 def dumpargs(*ap, **an):
 344     print("##",ap,an)
 345 
 346 def completer(text, state):
 347     U = [u for u in Users if u.startswith(text)]
 348     return U[state] if len(U)>state else None
 349 
 350 def lister(st, matches, maxlen):
 351     w = os.get_terminal_size().columns-2-len(prompt)
 352     sub = readline.get_line_buffer()
 353     res = " ".join(matches)[len(st):w+len(st)-1]
 354     print(" ".join(matches)[len(st):],prompt+sub,sep="\n",end="")
 355     sys.stdout.flush()
 356 
 357 def DoCalc():
 358     if "/" in C:
 359         Mkpenalty(set(C[1:C.index("/")]),set(C[C.index("/")+1:]))
 360     else:
 361         Mkpenalty(set(C[1:]))
 362 
 363 def DoTable():
 364     print(f'|||||||||||| {time.strftime("%F")} ||')
 365     print("|| № || '''Ник''' || '''Всего''' || '''Вовремя''' || '''Баллы''' || '''Оценка''' ||")
 366     Res = sorted(((Userscore(U),U) for U in Users),reverse=1)
 367     for i,((sc, a, f),U) in enumerate(Res,1):
 368         print("|| {} || {} || {} || {} || {} || {} ||".format(i,U,a,f,sc,grade(sc)))
 369 
 370 def UserCalc():
 371     sc, a, f = Userscore(C[1])
 372     print("{}: ({}/{}) {} = {}".format(C[1],a,f,sc,grade(sc) or "Неуд"))
 373 
 374 def DoTaskpasters():
 375     Taskpasters(C[0])
 376 
 377 def DoUserpastes():
 378     if len(C)<2:
 379         Userpastes(C[0])
 380     else:
 381         Userpastes(C[1], C[0])
 382 
 383 def ShowSource():
 384     print("{}({}):".format(*IDs[int(C[0])][:2]))
 385     print(Src[int(C[0])])
 386 
 387 def DoDiffsrc():
 388     Diffsrc(*C)
 389 
 390 def DoUsertasks():
 391     Usertasks(C[1])
 392 
 393 def DoTaskids():
 394     if len(C)<3:
 395         Taskids(C[1])
 396     else:
 397         Taskids(C[1],C[2])
 398 
 399 def DoUasage():
 400     print("Usage: {? [user] | ! [task] | id | user | task [user] | task user-or-id-1 user-or-id2}")
 401 
 402 readline.set_completer(completer)
 403 readline.set_completion_display_matches_hook(lister)
 404 
 405 HFile =  CF+".history.tmp"
 406 if os.path.isfile(HFile):
 407     readline.read_history_file(HFile)
 408 atexit.register(readline.write_history_file, HFile)
 409 cmd, prompt = "/", "GRR> "
 410 while True:
 411     C[:] = cmd.split()
 412     if len(C)>0 and C[0]=="/":
 413         DoCalc()
 414     elif len(C)==1 and C[0]=="!":
 415         Allpasters()
 416     elif len(C)==1 and C[0]=="?":
 417         DoTable()
 418     elif len(C)==2 and C[0]=="?" and C[1] in Users:
 419         UserCalc()
 420     elif len(C)==1 and C[0] in Tab:
 421         DoTaskpasters()
 422     elif len(C)==1 and C[0] in Users:
 423         DoUserpastes()
 424     elif len(C)==2 and C[0] in Tab and C[1] in Users:
 425         DoUserpastes()
 426     elif len(C)==1 and isID(C[0]):
 427         ShowSource()
 428     elif len(C)==3 and C[0] in Tab:
 429         DoDiffsrc()
 430     elif len(C)==2 and C[0]=="!" and C[1] in Users:
 431         DoUsertasks()
 432     elif len(C)==2 and C[0]=="!" and C[1] in Tab:
 433         DoTaskids()
 434     elif len(C)==3 and C[0]=="!" and C[1] in Tab and C[2] in Users:
 435         DoTaskids()
 436     elif C:
 437         DoUasage()
 438     try:
 439         cmd = input(prompt)
 440     except EOFError:
 441         break