Phase 2: full-voxel chunk storage, greedy mesher, and Babylon 3D viewer
api: chunk_sections table (per 16x16x16 section, base64-encoded u16 blockStateId array) and mesh_pointers table, additive to Phase 1's column-based chunk_columns/tile_pointers — 2D tile rendering keeps using the cheap column path unchanged. New "sections" WS message (backfill on chunk load + delta resend on flush, same "current state, not a diff" philosophy as columns) reuses the existing dirty-chunk Redis event, so one event now triggers the worker to re-render both the 2D tile and any 3D meshes for that chunk. New mesh-serving routes. worker: a from-scratch greedy mesher (per-axis 2D mask sweep + rectangle merge — the standard voxel-meshing technique, reimplemented from its public description, not copied from any codebase) producing a compact custom binary vertex buffer per non-empty section. Verified with unit tests, including one that specifically checks a uniform section collapses to exactly 6 merged quads rather than one quad per voxel face (the decisive signal that merging, not just per-voxel face emission, is actually happening). frontend: a barebones Babylon.js 3D viewer (/3d) that loads a fixed radius of chunks, parses the mesh binary format, and renders each section as its own mesh (no cross-section merging yet, no camera-based streaming yet — both reasonable follow-ups once there's a reason to optimize). End-to-end verified against live containers, including through the real mod-side Java WS client (see MCMapper-Mod's matching commit): a known half-solid section correctly round-trips to exactly 24 vertices / 36 indices at the mesh-serving endpoint, matching the "6 merged outer faces" the unit tests predict.
This commit is contained in:
Generated
+1
@@ -1732,6 +1732,7 @@ dependencies = [
|
||||
"aws-config",
|
||||
"aws-credential-types",
|
||||
"aws-sdk-s3",
|
||||
"base64",
|
||||
"image",
|
||||
"rayon",
|
||||
"redis",
|
||||
|
||||
@@ -14,6 +14,7 @@ image = { version = "0.25", default-features = false, features = ["png"] }
|
||||
aws-sdk-s3 = "1"
|
||||
aws-config = "1"
|
||||
aws-credential-types = "1"
|
||||
base64 = "0.22"
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
|
||||
@@ -38,6 +38,61 @@ pub async fn fetch_chunk_columns(
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
#[derive(sqlx::FromRow)]
|
||||
pub struct StoredSection {
|
||||
pub section_y: i32,
|
||||
pub blocks: String,
|
||||
}
|
||||
|
||||
pub async fn fetch_chunk_sections(
|
||||
pool: &PgPool,
|
||||
server_id: Uuid,
|
||||
dimension: i32,
|
||||
chunk_x: i32,
|
||||
chunk_z: i32,
|
||||
) -> anyhow::Result<Vec<StoredSection>> {
|
||||
let rows = sqlx::query_as::<_, StoredSection>(
|
||||
r#"SELECT section_y, blocks FROM chunk_sections
|
||||
WHERE server_id = $1 AND dimension = $2 AND x = $3 AND z = $4"#,
|
||||
)
|
||||
.bind(server_id)
|
||||
.bind(dimension)
|
||||
.bind(chunk_x)
|
||||
.bind(chunk_z)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn upsert_mesh_pointer(
|
||||
pool: &PgPool,
|
||||
server_id: Uuid,
|
||||
dimension: i32,
|
||||
chunk_x: i32,
|
||||
chunk_z: i32,
|
||||
section_y: i32,
|
||||
storage_key: &str,
|
||||
content_hash: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
sqlx::query(
|
||||
r#"INSERT INTO mesh_pointers (server_id, dimension, x, z, section_y, storage_key, content_hash, rendered_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, now())
|
||||
ON CONFLICT (server_id, dimension, x, z, section_y)
|
||||
DO UPDATE SET storage_key = excluded.storage_key, content_hash = excluded.content_hash, rendered_at = now()"#,
|
||||
)
|
||||
.bind(server_id)
|
||||
.bind(dimension)
|
||||
.bind(chunk_x)
|
||||
.bind(chunk_z)
|
||||
.bind(section_y)
|
||||
.bind(storage_key)
|
||||
.bind(content_hash)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn upsert_tile_pointer(
|
||||
pool: &PgPool,
|
||||
|
||||
+71
-1
@@ -1,4 +1,5 @@
|
||||
mod db;
|
||||
mod mesh;
|
||||
mod palette;
|
||||
mod render;
|
||||
mod storage;
|
||||
@@ -6,6 +7,7 @@ mod storage;
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::{Hash, Hasher};
|
||||
|
||||
use base64::{engine::general_purpose::STANDARD, Engine as _};
|
||||
use redis::streams::{StreamReadOptions, StreamReadReply};
|
||||
use redis::AsyncCommands;
|
||||
use render::{ColumnPixel, CpuRenderBackend, RenderBackend};
|
||||
@@ -119,7 +121,7 @@ async fn process_entry(
|
||||
let content_hash = format!("{:x}", hasher.finish());
|
||||
|
||||
let storage_key = format!("{server_id}/{dimension}/0/{chunk_x}/{chunk_z}.png");
|
||||
storage::put_tile(s3_client, &storage_key, png_bytes).await?;
|
||||
storage::put_object(s3_client, &storage_key, "image/png", png_bytes).await?;
|
||||
db::upsert_tile_pointer(
|
||||
pool,
|
||||
server_id,
|
||||
@@ -133,5 +135,73 @@ async fn process_entry(
|
||||
.await?;
|
||||
|
||||
println!("[worker] rendered tile {storage_key} ({} columns)", pixels.len());
|
||||
|
||||
mesh_chunk(pool, s3_client, server_id, dimension, chunk_x, chunk_z).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn decode_blocks(base64_blocks: &str) -> anyhow::Result<[u16; 4096]> {
|
||||
let bytes = STANDARD.decode(base64_blocks)?;
|
||||
if bytes.len() != 8192 {
|
||||
anyhow::bail!("expected 8192 bytes (4096 u16), got {}", bytes.len());
|
||||
}
|
||||
let mut blocks = [0u16; 4096];
|
||||
for (i, chunk) in bytes.chunks_exact(2).enumerate() {
|
||||
blocks[i] = u16::from_le_bytes([chunk[0], chunk[1]]);
|
||||
}
|
||||
Ok(blocks)
|
||||
}
|
||||
|
||||
async fn mesh_chunk(
|
||||
pool: &sqlx::PgPool,
|
||||
s3_client: &aws_sdk_s3::Client,
|
||||
server_id: Uuid,
|
||||
dimension: i32,
|
||||
chunk_x: i32,
|
||||
chunk_z: i32,
|
||||
) -> anyhow::Result<()> {
|
||||
let sections = db::fetch_chunk_sections(pool, server_id, dimension, chunk_x, chunk_z).await?;
|
||||
if sections.is_empty() {
|
||||
return Ok(()); // this server hasn't sent 3D data yet (Phase 2 mod support) — fine, no-op
|
||||
}
|
||||
|
||||
for section in sections {
|
||||
let blocks = match decode_blocks(§ion.blocks) {
|
||||
Ok(b) => b,
|
||||
Err(err) => {
|
||||
eprintln!(
|
||||
"[worker] skipping malformed section ({chunk_x},{chunk_z},{}): {err:#}",
|
||||
section.section_y
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let mesh_buf = mesh::mesh_section(&blocks);
|
||||
if mesh_buf.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mesh_bytes = mesh_buf.encode();
|
||||
|
||||
let mut hasher = DefaultHasher::new();
|
||||
mesh_bytes.hash(&mut hasher);
|
||||
let content_hash = format!("{:x}", hasher.finish());
|
||||
|
||||
let storage_key =
|
||||
format!("{server_id}/{dimension}/mesh/{chunk_x}/{chunk_z}/{}.bin", section.section_y);
|
||||
storage::put_object(s3_client, &storage_key, "application/octet-stream", mesh_bytes).await?;
|
||||
db::upsert_mesh_pointer(
|
||||
pool,
|
||||
server_id,
|
||||
dimension,
|
||||
chunk_x,
|
||||
chunk_z,
|
||||
section.section_y,
|
||||
&storage_key,
|
||||
&content_hash,
|
||||
)
|
||||
.await?;
|
||||
println!("[worker] rendered mesh {storage_key}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
use crate::palette::color_for;
|
||||
|
||||
const SIZE: i32 = 16;
|
||||
|
||||
/// Greedy-meshes a single 16x16x16 section into a flat vertex/index buffer. Sections are meshed
|
||||
/// independently (no merging across section/chunk boundaries in Phase 2 — a voxel at a section
|
||||
/// edge treats the neighbor as air even if an adjacent section has a solid block there), so a
|
||||
/// chunk with N non-empty sections becomes N small Babylon meshes rather than one combined mesh.
|
||||
/// That's a deliberate MVP simplification: it produces some redundant internal faces at section
|
||||
/// seams but avoids needing to fetch/hold neighboring sections just to mesh one.
|
||||
///
|
||||
/// This implements the standard "sweep each axis, build a 2D visibility mask per layer, greedily
|
||||
/// merge same-block runs into rectangles" technique (the general approach widely described for
|
||||
/// voxel engines, e.g. 0fps.net's "Meshing in a Minecraft Game" — reimplemented from that public
|
||||
/// description, not copied from any specific codebase).
|
||||
#[derive(Default)]
|
||||
pub struct MeshBuffers {
|
||||
pub positions: Vec<[f32; 3]>,
|
||||
pub normals: Vec<[f32; 3]>,
|
||||
pub colors: Vec<[f32; 3]>,
|
||||
pub indices: Vec<u32>,
|
||||
}
|
||||
|
||||
impl MeshBuffers {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.indices.is_empty()
|
||||
}
|
||||
|
||||
/// Binary layout consumed directly by the frontend (see frontend/src/public/js/mesh.js):
|
||||
/// `u32 vertexCount, u32 indexCount, f32[vertexCount*3] positions, f32[vertexCount*3]
|
||||
/// normals, f32[vertexCount*3] colors, u32[indexCount] indices` — all little-endian.
|
||||
pub fn encode(&self) -> Vec<u8> {
|
||||
let vertex_count = self.positions.len() as u32;
|
||||
let index_count = self.indices.len() as u32;
|
||||
let mut out = Vec::with_capacity(8 + (vertex_count as usize) * 36 + (index_count as usize) * 4);
|
||||
out.extend_from_slice(&vertex_count.to_le_bytes());
|
||||
out.extend_from_slice(&index_count.to_le_bytes());
|
||||
for p in &self.positions {
|
||||
for c in p {
|
||||
out.extend_from_slice(&c.to_le_bytes());
|
||||
}
|
||||
}
|
||||
for n in &self.normals {
|
||||
for c in n {
|
||||
out.extend_from_slice(&c.to_le_bytes());
|
||||
}
|
||||
}
|
||||
for c in &self.colors {
|
||||
for ch in c {
|
||||
out.extend_from_slice(&ch.to_le_bytes());
|
||||
}
|
||||
}
|
||||
for i in &self.indices {
|
||||
out.extend_from_slice(&i.to_le_bytes());
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
fn block_at(blocks: &[u16; 4096], x: i32, y: i32, z: i32) -> u16 {
|
||||
if x < 0 || x >= SIZE || y < 0 || y >= SIZE || z < 0 || z >= SIZE {
|
||||
return 0; // section boundary — treated as air, so boundary faces are always drawn
|
||||
}
|
||||
blocks[((y as usize) * 16 + z as usize) * 16 + x as usize]
|
||||
}
|
||||
|
||||
/// Maps (axis, layer, u, v) to a 3D voxel coordinate. axis 0 fixes x, 1 fixes y, 2 fixes z.
|
||||
fn axis_pos(axis: usize, layer: i32, u: i32, v: i32) -> (i32, i32, i32) {
|
||||
match axis {
|
||||
0 => (layer, u, v),
|
||||
1 => (u, layer, v),
|
||||
_ => (u, v, layer),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mesh_section(blocks: &[u16; 4096]) -> MeshBuffers {
|
||||
let mut buf = MeshBuffers::default();
|
||||
for axis in 0..3 {
|
||||
for &dir in &[-1i32, 1i32] {
|
||||
mesh_axis(blocks, axis, dir, &mut buf);
|
||||
}
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
fn mesh_axis(blocks: &[u16; 4096], axis: usize, dir: i32, buf: &mut MeshBuffers) {
|
||||
let mut mask = [[0u16; SIZE as usize]; SIZE as usize];
|
||||
|
||||
for layer in 0..SIZE {
|
||||
// Build the visibility mask for this layer: mask[u][v] = blockId if a face should be
|
||||
// drawn there (the voxel is solid and the neighbor in `dir` along `axis` is air/boundary).
|
||||
for u in 0..SIZE {
|
||||
for v in 0..SIZE {
|
||||
let (x, y, z) = axis_pos(axis, layer, u, v);
|
||||
let block = block_at(blocks, x, y, z);
|
||||
mask[u as usize][v as usize] = if block == 0 {
|
||||
0
|
||||
} else {
|
||||
let (ox, oy, oz) = offset_along_axis(axis, dir);
|
||||
let neighbor = block_at(blocks, x + ox, y + oy, z + oz);
|
||||
if neighbor == 0 { block } else { 0 }
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
let face_plane = if dir == 1 { layer + 1 } else { layer };
|
||||
greedy_merge_and_emit(&mut mask, axis, dir, face_plane, buf);
|
||||
}
|
||||
}
|
||||
|
||||
fn offset_along_axis(axis: usize, dir: i32) -> (i32, i32, i32) {
|
||||
match axis {
|
||||
0 => (dir, 0, 0),
|
||||
1 => (0, dir, 0),
|
||||
_ => (0, 0, dir),
|
||||
}
|
||||
}
|
||||
|
||||
fn greedy_merge_and_emit(
|
||||
mask: &mut [[u16; SIZE as usize]; SIZE as usize],
|
||||
axis: usize,
|
||||
dir: i32,
|
||||
face_plane: i32,
|
||||
buf: &mut MeshBuffers,
|
||||
) {
|
||||
let mut done = [[false; SIZE as usize]; SIZE as usize];
|
||||
|
||||
for u0 in 0..SIZE as usize {
|
||||
for v0 in 0..SIZE as usize {
|
||||
let block = mask[u0][v0];
|
||||
if block == 0 || done[u0][v0] {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Grow width along v.
|
||||
let mut v1 = v0 + 1;
|
||||
while v1 < SIZE as usize && mask[u0][v1] == block && !done[u0][v1] {
|
||||
v1 += 1;
|
||||
}
|
||||
|
||||
// Grow height along u, as long as the whole [v0, v1) run matches.
|
||||
let mut u1 = u0 + 1;
|
||||
'grow: while u1 < SIZE as usize {
|
||||
for v in v0..v1 {
|
||||
if mask[u1][v] != block || done[u1][v] {
|
||||
break 'grow;
|
||||
}
|
||||
}
|
||||
u1 += 1;
|
||||
}
|
||||
|
||||
for u in u0..u1 {
|
||||
for v in v0..v1 {
|
||||
done[u][v] = true;
|
||||
}
|
||||
}
|
||||
|
||||
emit_quad(axis, dir, face_plane, u0 as i32, v0 as i32, u1 as i32, v1 as i32, block, buf);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_quad(
|
||||
axis: usize,
|
||||
dir: i32,
|
||||
face_plane: i32,
|
||||
u0: i32,
|
||||
v0: i32,
|
||||
u1: i32,
|
||||
v1: i32,
|
||||
block: u16,
|
||||
buf: &mut MeshBuffers,
|
||||
) {
|
||||
let corners_uv = [(u0, v0), (u1, v0), (u1, v1), (u0, v1)];
|
||||
let base_index = buf.positions.len() as u32;
|
||||
|
||||
let normal = match (axis, dir) {
|
||||
(0, 1) => [1.0, 0.0, 0.0],
|
||||
(0, -1) => [-1.0, 0.0, 0.0],
|
||||
(1, 1) => [0.0, 1.0, 0.0],
|
||||
(1, -1) => [0.0, -1.0, 0.0],
|
||||
(2, 1) => [0.0, 0.0, 1.0],
|
||||
_ => [0.0, 0.0, -1.0],
|
||||
};
|
||||
let block_id = block >> 4;
|
||||
let block_meta = (block & 0xF) as u8;
|
||||
let [r, g, b] = color_for(block_id, block_meta);
|
||||
let color = [r as f32 / 255.0, g as f32 / 255.0, b as f32 / 255.0];
|
||||
|
||||
for (u, v) in corners_uv {
|
||||
let (x, y, z) = axis_pos(axis, face_plane, u, v);
|
||||
buf.positions.push([x as f32, y as f32, z as f32]);
|
||||
buf.normals.push(normal);
|
||||
buf.colors.push(color);
|
||||
}
|
||||
|
||||
// Two triangles per quad; flip winding by direction so both face orientations are at least
|
||||
// approximately correct. Backface culling is left off on the frontend material as the
|
||||
// safety net — see this module's doc comment and frontend/src/public/js/mesh.js.
|
||||
if dir == 1 {
|
||||
buf.indices.extend_from_slice(&[base_index, base_index + 1, base_index + 2, base_index, base_index + 2, base_index + 3]);
|
||||
} else {
|
||||
buf.indices.extend_from_slice(&[base_index, base_index + 2, base_index + 1, base_index, base_index + 3, base_index + 2]);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn empty_section_produces_no_geometry() {
|
||||
let blocks = [0u16; 4096];
|
||||
let mesh = mesh_section(&blocks);
|
||||
assert!(mesh.is_empty());
|
||||
assert_eq!(mesh.positions.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_voxel_produces_six_unmerged_quads() {
|
||||
let mut blocks = [0u16; 4096];
|
||||
blocks[((0 * 16 + 0) * 16 + 0) as usize] = (2 << 4) | 0; // grass at local (0,0,0)
|
||||
let mesh = mesh_section(&blocks);
|
||||
assert_eq!(mesh.positions.len(), 6 * 4, "6 faces x 4 verts");
|
||||
assert_eq!(mesh.indices.len(), 6 * 6, "6 faces x 2 tris x 3 indices");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_solid_section_collapses_to_six_merged_quads() {
|
||||
// If greedy merging weren't actually merging, this would instead emit thousands of
|
||||
// per-voxel quads (6 faces * 4096 voxels minus internal ones) — a full uniform section
|
||||
// must merge down to exactly one quad per outer face.
|
||||
let mut blocks = [0u16; 4096];
|
||||
for b in blocks.iter_mut() {
|
||||
*b = (1 << 4) | 0; // stone everywhere
|
||||
}
|
||||
let mesh = mesh_section(&blocks);
|
||||
assert_eq!(mesh.positions.len(), 6 * 4, "6 merged outer faces x 4 verts");
|
||||
assert_eq!(mesh.indices.len(), 6 * 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkerboard_layer_does_not_merge_across_gaps() {
|
||||
// A single y=0 layer, alternating solid/air in a checkerboard on x/z: no two solid
|
||||
// cells are adjacent, so the +y face mask can't merge anything — expect one quad per
|
||||
// solid cell for that face direction specifically.
|
||||
let mut blocks = [0u16; 4096];
|
||||
let mut solid_count = 0;
|
||||
for x in 0..16 {
|
||||
for z in 0..16 {
|
||||
if (x + z) % 2 == 0 {
|
||||
blocks[(z * 16 + x) as usize] = (1 << 4) | 0;
|
||||
solid_count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mesh = mesh_section(&blocks);
|
||||
// Just check the +y (top) face count via a targeted single-axis call.
|
||||
let mut buf = MeshBuffers::default();
|
||||
mesh_axis(&blocks, 1, 1, &mut buf);
|
||||
assert_eq!(buf.positions.len(), solid_count * 4);
|
||||
let _ = mesh; // silence unused warning if full mesh isn't otherwise inspected
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_round_trip_header() {
|
||||
let mut blocks = [0u16; 4096];
|
||||
blocks[0] = (2 << 4) | 0;
|
||||
let mesh = mesh_section(&blocks);
|
||||
let bytes = mesh.encode();
|
||||
let vertex_count = u32::from_le_bytes(bytes[0..4].try_into().unwrap());
|
||||
let index_count = u32::from_le_bytes(bytes[4..8].try_into().unwrap());
|
||||
assert_eq!(vertex_count as usize, mesh.positions.len());
|
||||
assert_eq!(index_count as usize, mesh.indices.len());
|
||||
let expected_len = 8 + vertex_count as usize * 36 + index_count as usize * 4;
|
||||
assert_eq!(bytes.len(), expected_len);
|
||||
}
|
||||
}
|
||||
@@ -25,12 +25,12 @@ pub async fn ensure_bucket(client: &Client) -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn put_tile(client: &Client, key: &str, bytes: Vec<u8>) -> anyhow::Result<()> {
|
||||
pub async fn put_object(client: &Client, key: &str, content_type: &str, bytes: Vec<u8>) -> anyhow::Result<()> {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(TILE_BUCKET)
|
||||
.key(key)
|
||||
.content_type("image/png")
|
||||
.content_type(content_type)
|
||||
.body(ByteStream::from(bytes))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
Reference in New Issue
Block a user