Bump engine submodule and merge vertex and param formats.

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "engine"]
 	path = engine
 	url = https://code.rozk.net/RK/rk_engine.git
+	branch = master

game/batch.py

@@ -13,54 +13,72 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from ctypes import c_ubyte, c_uint, c_void_p, addressof
+from ctypes import c_ubyte, c_ushort, c_void_p, POINTER, addressof
 
 from engine import (
-    buffer, INSTANCE_FLAG_SPAWNED, BATCH_MAX_SIZE, param_type, params_format,
+    buffer, INSTANCE_FLAG_SPAWNED, BATCH_MAX_SIZE, vertex_type, vertex_format,
     create_batch, fill_batch, draw_batch, destroy_batch)
 
-class Batch:
+_flags_t = c_ubyte
-    __slots__ = '_batch', '_static', 'vertices', 'size', 'max_size', 'flags', 'meshes', 'params', '_params', '__dict__'
+_flags_p = POINTER(_flags_t)
+_mesh_t = c_ushort
+_mesh_p = POINTER(_mesh_t)
 
-    def __init__(self, vertices, max_size, max_meshes, **params_decls):
+class Batch:
+    __slots__ = ('_batch', 'vertices', 'max_size', 'size',
+        'flags', '_flags', 'mesh', '_meshes', 'param', '_params', '_name', '__dict__')
+
+    def __init__(self, vertices, max_size, **params_decls):
         assert max_size <= BATCH_MAX_SIZE
         nparams = len(params_decls)
-        names = params_decls.keys()
-        formats = params_decls.values()
-        self._batch = create_batch(vertices._vertices, max_size, max_meshes, params_format(*formats))
-        self._static = False
-        self.vertices = vertices
-        self.size = 0
-        self.max_size = max_size
-        self.flags = buffer(c_ubyte, max_size)
-        self.meshes = buffer(c_uint, max_size)
-        self.params = tuple(map(lambda f: buffer(param_type(f), max_size), formats))
         if nparams:
-            self._params = (c_void_p * nparams)(*map(addressof, self.params))
+            self._batch = create_batch(vertices._vertices, max_size, vertex_format(*params_decls.values()))
         else:
-            self._params = None
+            self._batch = create_batch(vertices._vertices, max_size, None)
-        for name, value in zip(names, self.params):
+        self.vertices = vertices
+        self.max_size = max_size
+        self.size = 0
+        self.flags = buffer(_flags_t, max_size)
+        self._flags = _flags_p(self.flags)
+        self.mesh = buffer(_mesh_t, max_size)
+        self._meshes = _mesh_p(self.mesh)
+        if nparams:
+            self.param = tuple(map(lambda f: buffer(vertex_type(f), max_size), params_decls.values()))
+            self._params = (c_void_p * nparams)(*map(addressof, self.param))
+            self._name = dict(zip(params_decls.keys(), range(nparams)))
+            for name, value in zip(params_decls.keys(), self.param):
                 setattr(self, name, value)
+        else:
+            self.param = None
+            self._params = None
+            self._name = None
 
     def __del__(self):
         destroy_batch(self._batch)
 
-    def spawn(self, model, *params):
+    def spawn(self, flags, mesh, **params):
-        assert len(params) == len(self.params)
         index = self.size
         assert index < self.max_size
-        self.flags[index] = model.flags | INSTANCE_FLAG_SPAWNED
+        self.flags[index] = flags | INSTANCE_FLAG_SPAWNED
-        self.meshes[index] = model.mesh
+        self.mesh[index] = mesh
-        for self_params, param in zip(self.params, params):
+        for name, value in params.items():
-            self_params[index] = param
+            getattr(self, name)[index] = value
         self.size += 1
         return index
 
-    def make_static(self):
+    def freeze(self, flags = None, mesh = None, **params):
-        fill_batch(self._batch, self.size, self.flags, self.meshes, self._params)
+        if flags is not None:
-        self._static = True
+            self._flags = None if flags else _flags_p(self.flags)
+        if mesh is not None:
+            self._meshes = None if mesh else _mesh_p(self.mesh)
+        for name, value in params.items():
+            if value is not None:
+                index = self._name[name]
+                self._params[index] = None if value else addressof(self.param[index])
+
+    def fill(self):
+        fill_batch(self._batch, self.size, self._flags, self._meshes, self._params)
 
     def draw(self):
-        if not self._static:
+        fill_batch(self._batch, self.size, self._flags, self._meshes, self._params)
-            fill_batch(self._batch, self.size, self.flags, self.meshes, self._params)
         draw_batch(self._batch)

game/entity.py

@@ -16,5 +16,5 @@
 class Entity:
     __slots__ = 'index'
 
-    def __init__(self, batch, model, *params):
+    def __init__(self, batch, flags, mesh, **params):
-        self.index = batch.spawn(model, *params)
+        self.index = batch.spawn(flags, mesh, **params)

game/game.py

@@ -56,6 +56,7 @@ class PerfGroup(Group):
         self._name = name
 
     def __del__(self):
+        if self._count:
             avg = round(self._total / self._count, 2)
             print(self._name, "*", self._count,
                 ": min =", round(self._min, 2), ", max =", round(self._max, 2), ", avg =", avg, "(ms)")
@@ -75,8 +76,10 @@ class PerfGroup(Group):
 class TestEntity(Entity):
     __slots__ = 'translation', 'orientation', 'spawn_translation', 'spawn_orientation'
 
-    def __init__(self, batch, model, translation, orientation):
+    def __init__(self, batch, mesh, translation, orientation):
-        Entity.__init__(self, batch, model, translation, orientation)
+        Entity.__init__(self, batch, INSTANCE_FLAG_VISIBLE, mesh,
+            translation = translation,
+            orientation = orientation)
         self.translation = batch.translation[self.index]
         self.orientation = batch.orientation[self.index]
         self.spawn_translation = translation
@@ -126,15 +129,15 @@ def create_scene(keyboard, mouse):
 
     print("Building tiles...")
     tiles_texture = Texture(*archive.get_texture('tiles'))
-    tiles_vertices = Vertices(*archive.get_vertices('tiles'))
+    tiles_vertices = archive.get_vertices('tiles')
-    water_model = archive.get_model('water')
+    water_mesh = tiles_vertices.get_mesh('water')
-    sand_model = archive.get_model('sand')
+    sand_mesh = tiles_vertices.get_mesh('sand')
-    grass_model = archive.get_model('grass')
+    grass_mesh = tiles_vertices.get_mesh('grass')
-    forest_model = archive.get_model('forest')
+    forest_mesh = tiles_vertices.get_mesh('forest')
-    rock_model = archive.get_model('rock')
+    rock_mesh = tiles_vertices.get_mesh('rock')
-    mud_model = archive.get_model('mud')
+    mud_mesh = tiles_vertices.get_mesh('mud')
-    lava_model = archive.get_model('lava')
+    lava_mesh = tiles_vertices.get_mesh('lava')
-    tiles_batch = Batch(tiles_vertices, generated.size ** 2, 8, translation = PARAM_FORMAT_VEC3_SHORT)
+    tiles_batch = Batch(Vertices(*tiles_vertices), generated.size ** 2, translation = VERTEX_FORMAT_VEC3_SHORT)
 
     #TODO: generator & for real
     vc = generated.volcano_c
@@ -146,45 +149,49 @@ def create_scene(keyboard, mouse):
         if h == 0.0:
             continue
         if r > 0.0:
-            model = water_model
+            mesh = water_mesh
         elif h < 2.0:
-            model = sand_model
+            mesh = sand_mesh
         elif h < 180:
             if nz > 0.9:
                 if ny < -0.01 and nz > 0.93:
-                    model = forest_model
+                    mesh = forest_mesh
                 else:
-                    model = grass_model
+                    mesh = grass_mesh
             else:
-                model = rock_model
+                mesh = rock_mesh
         elif vd < vr - 3.0 and nz > 0.999:
-            model = lava_model
+            mesh = lava_mesh
         elif vd < vr + 2.0:
-            model = mud_model
+            mesh = mud_mesh
         elif vd < vr + 6.0 and nz < 0.67:
-            model = mud_model
+            mesh = mud_mesh
         else:
-            model = rock_model
+            mesh = rock_mesh
-        tiles_batch.spawn(model, vec3(float(((mx - 128) * 8) + 4), float(((127 - my) * 8) + 4), 0.0))
+        tiles_batch.spawn(INSTANCE_FLAG_VISIBLE, mesh,
-    tiles_batch.make_static()
+            translation = vec3(float(((mx - 128) * 8) + 4), float(((127 - my) * 8) + 4), 0.0))
+    tiles_batch.fill()
+    tiles_batch.freeze(flags = True, mesh = True, translation = True)
 
     tests_texture = Texture(*archive.get_texture('tests'))
-    tests_vertices = Vertices(*archive.get_vertices('tests'))
+    tests_vertices = archive.get_vertices('tests')
-    blob_model = archive.get_model('blob')
+    blob_mesh = tests_vertices.get_mesh('blob')
-    cube_model = archive.get_model('cube')
+    cube_mesh = tests_vertices.get_mesh('cube')
-    clouds_model = archive.get_model('clouds')
+    clouds_mesh = tests_vertices.get_mesh('clouds')
-    tests_batch = Batch(tests_vertices, 3, 3,
+    tests_batch = Batch(Vertices(*tests_vertices), 3,
-        translation = PARAM_FORMAT_VEC3_FLOAT,
+        translation = VERTEX_FORMAT_VEC3_FLOAT,
-        orientation = PARAM_FORMAT_MAT3_INT10 | PARAM_FORMAT_NORMALIZE)
+        orientation = VERTEX_FORMAT_MAT3_INT10 | VERTEX_FORMAT_NORMALIZE)
     blob_forward = math.vec3_normalize((sun_direction[0], sun_direction[1], 0.0))
     blob_right = math.vec3_cross(blob_forward, math.vec3_up)
     blob_orientation = mat3(vec3(*blob_right), vec3(*blob_forward), vec3_up)
-    blob = TestEntity(tests_batch, blob_model, vec3(-100.0, -500.0, 0.0), blob_orientation)
+    blob = TestEntity(tests_batch, blob_mesh, translation = vec3(-100.0, -500.0, 0.0), orientation = blob_orientation)
-    cube = TestEntity(tests_batch, cube_model, vec3(100.0, -500.0, 0.0), mat3_identity)
+    cube = TestEntity(tests_batch, cube_mesh, translation = vec3(100.0, -500.0, 0.0), orientation = mat3_identity)
-    clouds = TestEntity(tests_batch, clouds_model, vec3(0.0, 0.0, 32.0), mat3_identity)
+    clouds = TestEntity(tests_batch, clouds_mesh, translation = vec3(0.0, 0.0, 32.0), orientation = mat3_identity)
+    tests_batch.fill()
+    tests_batch.freeze(flags = True, mesh = True)
 
-    sea_polar_textures = sea.load_polar_textures(('data/sea_bump1.png', 'data/sea_bump2.png'))
+    sea_polar = sea.load_polar_textures(('data/sea_bump1.png', 'data/sea_bump2.png'))
-    sea_detail_texture = sea.load_detail_texture('data/sea_bump.png')
+    sea_detail = sea.load_detail_texture('data/sea_bump.png')
     sea_triangles = sea.sea_triangles(64, proj_far_z - 0.1, proj_ratio)
 
     assert tiles_shader.u_height_sampler == tests_shader.u_height_sampler
@@ -206,8 +213,7 @@ def create_scene(keyboard, mouse):
                         PerfGroup('tests_batch',
                             DrawNode(tests_batch))))),
             FuncNode(update_sea, (camera, sea_phase)),
-            TextureGroup(
+            TextureGroup({sea_shader.u_polar_sampler: sea_polar, sea_shader.u_detail_sampler: sea_detail},
-                {sea_shader.u_polar_sampler: sea_polar_textures, sea_shader.u_detail_sampler: sea_detail_texture},
                 ShaderGroup(sea_shader,
                     InputNode(sea_shader, camera, environment, sea_phase),
                     PerfGroup('sea_triangles',
@@ -219,14 +225,14 @@ def loop(display):
     mouse = Mouse(events, wheel = 60, wheel_min = 20)
     scene = create_scene(keyboard, mouse)
     print("Running... Ctrl+c to quit")
-    time = Time()
     try:
+        time = Time()
         while not keyboard.quit:
             events.update()
-            time.update()
             clear_buffer(True, True, True)
             scene.draw(time)
             swap_buffers(display)
+            time.update()
     except KeyboardInterrupt:
         pass
 

game/obj2rkar.py

@@ -22,7 +22,7 @@ from itertools import starmap, chain, repeat
 from engine import *
 
 from game.math import float_s8, vec3_srgb8a8
-from game.resources import load_png, TextureData, VerticesData, ModelData, Archive
+from game.resources import load_png, TextureData, VerticesData, Archive
 
 _texture_flags = TEXTURE_FLAG_MIPMAPS | TEXTURE_FLAG_MIN_LINEAR | TEXTURE_FLAG_MAG_NEAREST
 
@@ -195,7 +195,7 @@ class ObjArchive(Archive):
 
         vertices = []
         indices = []
-        models = {}
+        meshes = {}
         for name, mesh in sorted(objects.items()):
             offset = len(indices)
             assert offset < 65536
@@ -213,28 +213,26 @@ class ObjArchive(Archive):
             assert count < 65536
             vertices.extend(mesh_vertices)
             indices.extend(mesh_indices)
-            models[name] = (offset, count)
+            meshes[name] = (offset, count)
 
         name = str(objpath.stem)
         assert name not in self.vertices_db.keys()
         #TODO: move to math
         def pack_10(_x):
-            return round(_x * (512.0 if _x < 0.0 else 511.0)) & 1023
+            assert _x >= -1.0 and _x <= 1.0
+            return round(_x * 511.0) & 1023
+            # return ((round(_x * 1023.0) - 1) // 2) & 1023
         def pack_u10(_x):
-            return round(_x * 1023.0) & 1023
+            assert _x >= 0.0 and _x <= 1.0
+            return round(_x * 1023.0)
         def pack_vertex(_px, _py, _pz, _nx, _ny, _nz, _s, _t, _tl):
             n = (pack_10(_nz) << 20) | (pack_10(_ny) << 10) | pack_10(_nx)
             t = ((_tl & 1023) << 20) | (pack_u10(_t) << 10) | pack_u10(_s)
             return struct.pack('fffII', _px, _py, _pz, n, t)
         self.vertices_db[name] = VerticesData(name,
             array('B', b''.join(starmap(pack_vertex, vertices))),
-            array('H', indices))
+            array('H', indices),
-        for name, (offset, count) in sorted(models.items()):
+            meshes)
-            if name[0] != '_':
-                print("Storing", name)
-                assert name not in self.models_db.keys()
-                self.models_db[name] = ModelData(
-                    name, INSTANCE_FLAG_VISIBLE, offset | count << 16)
 
 if __name__ == '__main__':
     if len(sys.argv) < 3:

game/resources.py

@@ -13,6 +13,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from itertools import chain
 import struct
 from array import array
 from pathlib import Path
@@ -48,7 +49,7 @@ def _read_struct(file, format):
     _read_magic(file, b'RK')
     return struct.unpack(format, data)
 
-def _write_struct(file, format, elems):
+def _write_struct(file, format, *elems):
     assert format
     data = struct.pack(format, *elems)
     size = file.write(data)
@@ -56,7 +57,7 @@ def _write_struct(file, format, elems):
     _write_magic(file, b'RK')
 
 def _read_string(file):
-    length, = _read_struct(file, 'B')
+    length, = _read_struct(file, 'I')
     assert length
     data = file.read(length)
     assert len(data) == length
@@ -66,7 +67,7 @@ def _read_string(file):
 def _write_string(file, string):
     data = bytes(string, encoding='ascii')
     assert data and len(data) < 256
-    _write_struct(file, 'B', (len(data),))
+    _write_struct(file, 'I', len(data))
     size = file.write(data)
     assert size == len(data)
     _write_magic(file, b'RK')
@@ -96,7 +97,7 @@ def _read_blob(file):
 
 def _write_blob(file, array):
     assert array
-    _write_struct(file, 'II', (ord(array.typecode), len(array)))
+    _write_struct(file, 'II', ord(array.typecode), len(array))
     array.tofile(file)
     _write_magic(file, b'RK')
 
@@ -124,7 +125,7 @@ class TextureData:
 
     @classmethod
     def from_archive(cls, file):
-        _read_magic(file, b'TX')
+        _read_magic(file, b'TXTR')
         name = _read_string(file)
         format, width, height, nlevels, flags = _read_struct(file, 'IIIII')
         pixels = _read_blob(file)
@@ -133,70 +134,61 @@ class TextureData:
         return cls(name, format, width, height, nlevels, flags, pixels)
 
     def to_archive(self, file):
-        _write_magic(file, b'TX')
+        _write_magic(file, b'TXTR')
         _write_string(file, self.name)
-        _write_struct(file, 'IIIII', (self.format, self.width, self.height, self.nlevels, self.flags))
+        _write_struct(file, 'IIIII', self.format, self.width, self.height, self.nlevels, self.flags)
         _write_blob(file, self.pixels)
 
 class VerticesData:
-    __slots__ = 'name', 'vertices', 'indices'
+    __slots__ = 'name', 'vertices', 'indices', 'meshes', 'meshes_db'
 
-    def __init__(self, name, vertices, indices):
+    def __init__(self, name, vertices, indices, meshes):
         if len(vertices) % VERTEX_SIZE != 0:
             raise RuntimeError("Vertex format mismatch!")
         self.name = name
         self.vertices = vertices
         self.indices = indices
+        self.meshes = meshes.values()
+        self.meshes_db = dict(zip(meshes.keys(), range(len(meshes))))
 
     def __iter__(self):
         yield VERTEX_FORMAT
         yield len(self.vertices) // VERTEX_SIZE
         yield self.vertices
         yield self.indices
+        yield array('I', chain.from_iterable(self.meshes))
+
+    def get_mesh(self, name):
+        return self.meshes_db[name]
 
     @classmethod
     def from_archive(cls, file):
-        _read_magic(file, b'VT')
+        _read_magic(file, b'VERT')
         name = _read_string(file)
-        nvertices, nindices = _read_struct(file, 'II')
+        nvertices, nindices, nmeshes = _read_struct(file, 'III')
         vertices = _read_array(file, 'B', nvertices * VERTEX_SIZE)
         indices = _read_array(file, 'H', nindices)
-        return cls(name, vertices, indices)
+        names = [_read_string(file) for _ in range(nmeshes)]
+        meshes = [_read_struct(file, 'II') for _ in range(nmeshes)]
+        return cls(name, vertices, indices, dict(zip(names, meshes)))
 
     def to_archive(self, file):
-        _write_magic(file, b'VT')
+        _write_magic(file, b'VERT')
         _write_string(file, self.name)
-        _write_struct(file, 'II', (len(self.vertices) // VERTEX_SIZE, len(self.indices)))
+        _write_struct(file, 'III', len(self.vertices) // VERTEX_SIZE, len(self.indices), len(self.meshes_db))
         _write_array(file, self.vertices)
         _write_array(file, self.indices)
-
+        for name in self.meshes_db.keys():
-class ModelData:
+            _write_string(file, name)
-    __slots__ = 'name', 'flags', 'mesh'
+        for mesh in self.meshes:
-
+            _write_struct(file, 'II', *mesh)
-    def __init__(self, name, flags, mesh):
-        self.name = name
-        self.flags = flags
-        self.mesh = mesh
-
-    @classmethod
-    def from_archive(cls, file):
-        _read_magic(file, b'MD')
-        name = _read_string(file)
-        flags, mesh = _read_struct(file, 'II')
-        return ModelData(name, flags, mesh)
-
-    def to_archive(self, file):
-        _write_magic(file, b'MD')
-        _write_string(file, self.name)
-        _write_struct(file, 'II', (self.flags, self.mesh))
 
 class Archive:
-    __slots__ = 'textures_db', 'vertices_db', 'models_db'
+    __slots__ = 'textures_db', 'vertices_db'
 
-    def __init__(self, textures_db = None, vertices_db = None, models_db = None):
+    def __init__(self, textures_db = None, vertices_db = None):
         self.textures_db = textures_db or {}
         self.vertices_db = vertices_db or {}
-        self.models_db = models_db or {}
 
     def get_texture(self, name):
         return self.textures_db[name]
@@ -204,36 +196,27 @@ class Archive:
     def get_vertices(self, name):
         return self.vertices_db[name]
 
-    def get_model(self, name):
-        return self.models_db[name]
-
     @classmethod
     def from_archive(cls, file):
         textures_db = {}
         vertices_db = {}
-        models_db = {}
         _read_magic(file, b'RKAR')
-        ntextures, nvertices, nmodels = _read_struct(file, 'III')
+        ntextures, nvertices = _read_struct(file, 'II')
         for _ in range(ntextures):
             data = TextureData.from_archive(file)
             textures_db[data.name] = data
         for _ in range(nvertices):
             data = VerticesData.from_archive(file)
             vertices_db[data.name] = data
-        for _ in range(nmodels):
+        return cls(textures_db, vertices_db)
-            data = ModelData.from_archive(file)
-            models_db[data.name] = data
-        return cls(textures_db, vertices_db, models_db)
 
     def to_archive(self, file):
         _write_magic(file, b'RKAR')
-        _write_struct(file, 'III', (len(self.textures_db), len(self.vertices_db), len(self.models_db)))
+        _write_struct(file, 'II', len(self.textures_db), len(self.vertices_db))
         for _, data in self.textures_db.items():
             data.to_archive(file)
         for _, data in self.vertices_db.items():
             data.to_archive(file)
-        for _, data in self.models_db.items():
-            data.to_archive(file)
 
     @classmethod
     def load(cls, filename):

game/vertices.py

@@ -18,8 +18,8 @@ from engine import create_vertices, destroy_vertices
 class Vertices:
     __slots__ = '_vertices'
 
-    def __init__(self, format, nvertices, vertices, indices):
+    def __init__(self, format, nvertices, vertices, indices, meshes):
-        self._vertices = create_vertices(format, nvertices, vertices, indices)
+        self._vertices = create_vertices(format, nvertices, vertices, indices, meshes)
 
     def __del__(self):
         destroy_vertices(self._vertices)