Save chunks in db

One of the changes introduced for this is that multiple chunks and
diffs are now represented by dicts, not lists.

chunks.py

@@ -25,13 +25,21 @@ class ChunkDiff():
 		diff = cls()
 		diff._chars = {int(i): v for i, v in d.items()}
 		return diff
-		#self._chars = d.copy()
+	
+	def to_dict(self):
+		return self._chars
+		#return self._chars.copy()
 	
 	@classmethod
 	def from_string(cls, s):
 		diff = cls()
-		#for c in string
+		for i, char in enumerate(s):
-		pass
+			diff._chars[i] = char
+		return diff
+	
+	def to_string(self):
+		s = "".join(self._chars.get(i, " ") for i in range(CHUNK_WIDTH*CHUNK_HEIGHT))
+		return s
 	
 	def copy(self):
 		return ChunkDiff.from_dict(self.to_dict().copy())
@@ -41,10 +49,6 @@ class ChunkDiff():
 		newdiff.apply(diff)
 		return newdiff
 	
-	def to_dict(self):
-		return self._chars
-		#return self._chars.copy()
-	
 	def set(self, x, y, character):
 		pos = x+y*CHUNK_WIDTH
 		self._chars[pos] = character
@@ -60,8 +64,7 @@ class ChunkDiff():
 			self._chars[i] = c
 	
 	def lines(self):
-		d = self._chars
+		s = self.to_string()
-		s = "".join(d.get(i, " ") for i in range(CHUNK_WIDTH*CHUNK_HEIGHT))
 		return [s[i:i+CHUNK_WIDTH] for i in range(0, CHUNK_WIDTH*CHUNK_HEIGHT, CHUNK_WIDTH)]
 	
 	def empty(self):
@@ -83,19 +86,18 @@ class ChunkDiff():
 
 def jsonify_diffs(diffs):
 	ddiffs = []
-	for dchunk in diffs:
+	for pos, diff in diffs.items():
-		pos = dchunk[0]
+		ddiff = diff.to_dict()
-		ddiff = dchunk[1].to_dict()
 		ddiffs.append((pos, ddiff))
 	
 	return ddiffs
 
 def dejsonify_diffs(ddiffs):
-	diffs = []
+	diffs = {}
 	for dchunk in ddiffs:
 		pos = Position(dchunk[0][0], dchunk[0][1])
 		diff = ChunkDiff.from_dict(dchunk[1])
-		diffs.append((pos, diff))
+		diffs[pos] = diff
 	
 	return diffs
 
@@ -114,6 +116,15 @@ class Chunk():
 		
 		self.last_modified = 0
 	
+	@classmethod
+	def from_string(cls, s):
+		chunk = cls()
+		chunk._content = ChunkDiff.from_string(s)
+		return chunk
+	
+	def to_string(self):
+		return self.as_diff().to_string()
+	
 	def set(self, x, y, character):
 		self._modifications.set(x, y, character)
 		self.touch()
@@ -191,9 +202,7 @@ class ChunkPool():
 		return chunk
 	
 	def apply_diffs(self, diffs):
-		for dchunk in diffs:
+		for pos, diff in diffs.items():
-			pos = dchunk[0]
-			diff = dchunk[1]
 			chunk = self.get(pos) or self.create(pos)
 			#chunk = self.load(pos)
 			
@@ -201,9 +210,7 @@ class ChunkPool():
 				chunk.apply_diff(diff)
 	
 	def commit_diffs(self, diffs):
-		for dchunk in diffs:
+		for pos, diff in diffs.items():
-			pos = dchunk[0]
-			diff = dchunk[1]
 			chunk = self.get(pos) or self.create(pos)
 			#chunk = self.load(pos)
 			
@@ -211,11 +218,11 @@ class ChunkPool():
 				chunk.commit_diff(diff)
 	
 	def commit_changes(self):
-		changes = []
+		changes = {}
 		
 		for pos, chunk in self._chunks.items():
 			if chunk.modified():
-				changes.append((pos, chunk.get_changes()))
+				changes[pos] = chunk.get_changes()
 				chunk.commit_changes()
 		
 		return changes

clientchunkpool.py

@@ -31,7 +31,7 @@ class ClientChunkPool(ChunkPool):
 	def save_changes(self):
 		diffs = self.commit_changes()
 		# filter out empty changes/chunks
-		diffs = [dchunk for dchunk in diffs if not dchunk[1].empty()]
+		diffs = {pos: diff for pos, diff in diffs.items() if not diff.empty()}
 		
 		if diffs:
 			self._client.send_changes(diffs)

dbchunkpool.py

@@ -2,8 +2,8 @@ import sqlite3
 import time
 import threading
 
-from chunks import ChunkPool
+from chunks import ChunkPool, Chunk
-from utils import Position
+from utils import Position, CHUNK_WIDTH, CHUNK_HEIGHT
 
 class ChunkDB():
 	"""
@@ -13,6 +13,8 @@ class ChunkDB():
 	def __init__(self, filename):
 		self.dbfilename = filename
 		
+		self._create_table()
+	
 	def transaction(func):
 		def wrapper(self, *args, **kwargs):
 			con = sqlite3.connect(self.dbfilename)
@@ -24,14 +26,56 @@ class ChunkDB():
 		
 		return wrapper
 	
+	@transaction
+	def _create_table(self, con):
+		cur = con.cursor()
+		cur.execute(("CREATE TABLE IF NOT EXISTS chunks ("
+		             "x INTEGER NOT NULL, "
+		             "y INTEGER NOT NULL, "
+		             "content TEXT, "
+		             "PRIMARY KEY (x, y)"
+		             ")"))
+	
 	@transaction
 	def save_many(self, con, chunks):
-		print("save_many")
+		lchunks = ChunkDB.chunks_to_list(chunks)
+		con.executemany("INSERT OR REPLACE INTO chunks VALUES (?, ?, ?)", lchunks)
 	
 	@transaction
 	def load_many(self, con, coords):
-		print("load_many")
+		cur = con.cursor()
-		return [(coor, None) for coor in coords]
+		results = []
+		
+		for pos in coords:
+			cur.execute("SELECT * FROM chunks WHERE x=? AND y=?", pos)
+			results.extend(cur.fetchall())
+		
+		results = ChunkDB.list_to_chunks(results)
+		return results
+	
+	@transaction
+	def remove_empty(self, con):
+		con.execute("DELETE FROM chunks WHERE content=?", (" "*CHUNK_WIDTH*CHUNK_HEIGHT,))
+	
+	@staticmethod
+	def list_to_chunks(l):
+		chunks = {}
+		
+		for item in l:
+			pos = Position(item[0], item[1])
+			chunk = Chunk.from_string(item[2])
+			chunks[pos] = chunk
+		
+		return chunks
+	
+	@staticmethod
+	def chunks_to_list(chunks):
+		l = []
+		
+		for pos, chunk in chunks.items():
+			l.append((pos[0], pos[1], chunk.to_string()))
+		
+		return l
 
 class DBChunkPool(ChunkPool):
 	"""
@@ -42,8 +86,8 @@ class DBChunkPool(ChunkPool):
 		super().__init__()
 		self._chunkdb = ChunkDB(filename)
 		
-		self.save_period = 10 # save and clean up every minute
+		self.save_period = 60 # save and clean up every minute
-		self.max_age = 20 # ca. one minute until a chunk is unloaded again
+		self.max_age = 60 # ca. one minute until a chunk is unloaded again
 		
 		self.save_thread = threading.Thread(
 			target=self.perodic_save,
@@ -55,27 +99,23 @@ class DBChunkPool(ChunkPool):
 	def save_changes(self):
 		diffs = self.commit_changes()
 		
-		changed_chunks = []
+		changed_chunks = {}
-		for dchunk in diffs:
+		for pos, diff in diffs.items():
-			pos = dchunk[0]
 			chunk = self.get(pos)
-			changed_chunks.append((pos, chunk))
+			changed_chunks[pos] = chunk
 		
 		self._chunkdb.save_many(changed_chunks)
 	
 	def load(self, pos):
-		print("Loading individual chunk...")
 		raise Exception
 	
 	def load_list(self, coords):
-		print("Loading chunk list...")
 		to_load = [pos for pos in coords if pos not in self._chunks]
 		chunks = self._chunkdb.load_many(to_load)
-		for dchunk in chunks:
+		
-			pos = dchunk[0]
+		for pos in to_load:
-			chunk = dchunk[1]
+			if pos in chunks:
-			if chunk:
+				self.set(pos, chunks.get(pos))
-				self.set(pos, chunk)
 			else:
 				self.create(pos)
 	
@@ -84,21 +124,20 @@ class DBChunkPool(ChunkPool):
 			time.sleep(self.save_period)
 			
 			with self:
-				print("BEFORE:::")
-				self.print_chunks()
-				
 				self.save_changes()
 				
 				# unload old chunks
 				now = time.time()
-				for pos, chunk in self._chunks.items():
-					print(f"p{pos} :: t{now} :: m{chunk.last_modified} :: a{chunk.age(now)}")
 				self.clean_up(condition=lambda pos, chunk: chunk.age(now) > self.max_age)
 	
-				print("AFTER:::")
+	def remove_empty(self):
-				self.print_chunks()
+		self._chunkdb.remove_empty()
+	
+	def _get_min_max(self):
+		"""
+		Meant for debugging.
+		"""
 		
-	def get_min_max(self):
 		minx = min(pos.x for pos in self._chunks)
 		maxx = max(pos.x for pos in self._chunks)
 		miny = min(pos.y for pos in self._chunks)
@@ -106,9 +145,13 @@ class DBChunkPool(ChunkPool):
 		
 		return minx, maxx, miny, maxy
 	
-	def print_chunks(self):
+	def _print_chunks(self):
+		"""
+		Meant for debugging.
+		"""
+		
 		if self._chunks:
-			minx, maxx, miny, maxy = self.get_min_max()
+			minx, maxx, miny, maxy = self._get_min_max()
 			sizex, sizey = maxx - minx + 1, maxy - miny + 1
 			print("┌" + "─"*sizex*2 + "┐")
 			for y in range(miny, maxy + 1):

server.py

@@ -12,14 +12,16 @@ from chunks import ChunkPool
 
 class WotServer(WebSocket):
 	def handle_request_chunks(self, coords):
-		diffs = []
+		diffs = {}
-		with self.pool as pool:
 		coords = [Position(coor[0], coor[1]) for coor in coords]
+		
+		with self.pool as pool:
 			pool.load_list(coords)
 			
 			for pos in coords:
 				chunk = pool.get(pos)
-				diffs.append((pos, chunk.as_diff()))
+				diff = chunk.as_diff()
+				diffs[pos] = diff
 				
 				self.loaded_chunks.add(pos)
 		
@@ -38,16 +40,17 @@ class WotServer(WebSocket):
 		
 		# check whether changes are correct (exclude certain characters)
 		# if not correct, send corrections them back to sender
-		legitimate_diffs = []
+		legitimate_diffs = {}
-		illegitimate_diffs = []
+		illegitimate_diffs = {}
-		for dchunk in diffs:
+		for pos, diff in diffs.items():
-			if dchunk[1].legitimate():
+			if diff.legitimate():
-				legitimate_diffs.append(dchunk)
+				legitimate_diffs[pos] = diff
 			else:
-				illegitimate_diffs.append(dchunk)
+				illegitimate_diffs[pos] = diff
 		
 		if legitimate_diffs:
 			with self.pool as pool:
+				pool.load_list(legitimate_diffs.keys())
 				pool.apply_diffs(legitimate_diffs)
 			
 			for client in self.clients:
@@ -61,23 +64,19 @@ class WotServer(WebSocket):
 			self.sendMessage(json.dumps(message))
 	
 	def reverse_diffs(self, diffs):
-		coords = [dchunk[0] for dchunk in diffs]
-		
 		with self.pool as pool:
-			pool.load_list(coords)
+			pool.load_list(diffs.keys())
 			
-			reverse_diffs = []
+			reverse_diffs = {}
-			for dchunk in diffs:
+			for pos, diff in diffs.items():
-				pos = dchunk[0]
-				diff = dchunk[1]
 				chunk = pool.get(pos)
 				reverse_diff = diff.diff(chunk.as_diff())
-				reverse_diffs.append((pos, reverse_diff))
+				reverse_diffs[pos] = reverse_diff
 		
 		return reverse_diffs
 	
 	def send_changes(self, diffs):
-		diffs = [dchunk for dchunk in diffs if dchunk[0] in self.loaded_chunks]
+		diffs = {pos: diff for pos, diff in diffs.items() if pos in self.loaded_chunks}
 		ddiffs = jsonify_diffs(diffs)
 		
 		if ddiffs:
@@ -146,6 +145,8 @@ def main(argv):
 		print("")
 		print("Saving recent changes.")
 		WotServer.pool.save_changes()
+		print("Cleaning up empty chunks from db.")
+		WotServer.pool.remove_empty()
 		print("Stopped.")
 
 if __name__ == "__main__":