Improve buffer efficiency

master
Ringo Wantanabe 6 years ago
parent f6a8da49c3
commit 1a9c4a5698
No known key found for this signature in database
GPG Key ID: C1C1CD34CF2907B2

@ -0,0 +1 @@
pylzma

@ -1,3 +1,4 @@
import sys
import os
import os.path
@ -11,7 +12,7 @@ import datetime
import json
import re
import time
import threading
from threading import Thread, Lock
import binascii
import pylzma
from cffi import FFI
@ -76,8 +77,14 @@ class StatBuffer:
SLV_NOUI = 3 #Remove all user inputed information (sub, com,name,trip, file info)
SLV_HIGH = 0xFF #Keep only post number
def __init__(self):
self._mutex = Lock()
pass
def _lock(self):
self._mutex.acquire()
def _unlock(self):
self._mutex.release()
def _decode(self, data):
if(isinstance(data, int)):
return data
@ -128,75 +135,132 @@ class StatBuffer:
class MemoryBuffer(StatBuffer):
def __init__(self, level):
super().__init__()
self.lvl = level
self.store = list()
def write(self, post):
super()._lock()
data = super()._encode(post, self.lvl)
self.store.append(data)
super()._unlock()
def raw(self):
super()._lock()
try:
return json.dumps(self.store)
finally:
super()._unlock()
def clear(self):
super()._lock()
self.store = list()
super()._unlock()
def length(self):
super()._lock()
try:
return len(self.store)
finally:
super()._unlock()
def read(self):
super()._lock()
try:
base = super()
return list(base._decode(d) for d in self.store)
finally:
super()._unlock()
def findMax(self):
mx=0
for ent in self.store:
post = super()._decode(ent)
if(post["no"]>mx): mx = post["no"]
return mx
super()._lock()
try:
if len(self.store)<1: return 0
return super()._decode(self.store[-1])["no"]
finally:
super()._unlock()
def readno(self, floor):
super()._lock()
posts = list()
nl = len(self.store)-1
while(nl>=0):
entry = super()._decode(self.store[nl])
if(entry["no"]<=floor): break
posts.append(ent)
nl-=1
super()._unlock()
return posts
class FileBuffer(StatBuffer):
def __init__(self, fn, level):
super().__init__()
self.lvl = level
self.file = open(fn, "a+b")
def write(self, post):
super()._lock()
data = super()._encode(post, self.lvl)
self.file.write(struct.pack("I", len(data)))
self.file.write(data)
def read(self):
self.file.seek(0)
posts = list()
self.file.write(struct.pack("I", len(data)))
super()._unlock()
def _readentry(self):
self.file.seek(-4,1)
lentr = self.file.read(4)
while( lentr != None):
if(len(lentr)<4): break
if len(lentr)<4: return None
tl = struct.unpack("I", lentr)[0]
#pprint.pprint(tl)
posts.append(super()._decode(self.file.read(tl)))
self.file.seek(-(tl+4), 1)
ret = super()._decode(self.file.read(tl))
self.file.seek(-tl,1)
return ret
def _skipentry(self):
self.file.seek(-4,1)
lentr = self.file.read(4)
if len(lentr)<4: return False
tl = struct.unpack("I", lentr)[0]
self.file.seek(-(tl+4),1)
return True
def read(self):
super()._lock()
posts = list()
ent = self._skipentry()
while self.file.tell()>0 and ent!=None:
posts.append(ent)
ent = self._readentry()
self.file.seek(0,2)
super()._unlock()
return posts
def length(self):
self.file.seek(0)
maxln = 0
lentr=self.file.read(4)
while( lentr != None):
if(len(lentr)<4): break
tl = struct.unpack("I", lentr)[0]
self.file.seek(tl, 1)
maxln+=1
lentr=self.file.read(4)#wonderful language lmao
super()._lock()
ent = self._skipentry()
maxl =0
while self.file.tell()>0 and ent:
maxl+=1
ent = self._skipentry()
self.file.seek(0,2)
return maxln
super()._unlock()
return maxl
def close(self):
super()._lock()
self.file.close()
super()._unlock()
def clear(self):
super()._lock()
self.file.truncate()
super()._unlock()
def findMax(self):
self.file.seek(0)
mx=0
lentr=self.file.read(4)
while( lentr != None):
if(len(lentr)<4): break
tl = struct.unpack("I", lentr)[0]
post = super()._decode(self.file.read(tl))
if(post["no"] > mx): mx = post["no"]
lentr=self.file.read(4)#wonderful language lmao
super()._lock()
try:
if(self.file.tell()<1): return 0
sk = self._readentry()
self.file.seek(0,2)
return mx
if sk!=None: return sk["no"]
else: return 0
finally:
super()._unlock()
def readno(self, floor):
super()._lock()
posts = list()
ent = self._skipentry()
while self.file.tell()>0 and ent!=None:
if(ent["no"]<=floor): break
posts.append(ent)
ent = self._readentry()
self.file.seek(0,2)
super()._unlock()
return posts
def parse_post(post):
res = dict()

Binary file not shown.
Loading…
Cancel
Save