// GPU color/texture transfer for remeshing demo // // Remeshed atlas is rasterized in UV space, the triangle edges are expanded to cover extra gutter space. // The fragment shader traces rays against the original mesh (falling back to closest point on miss). // The results are shaded using a simplified version of the original material, and written to the output texture. // Texels that were not rasterized are infilled afterwards using the mips of the resulting texture. // When painting vertex colors, we directly rasterize points for target vertex/triangles and trace rays/eval materials as above. import * as THREE from 'three'; import { MeshBVH, BVHShaderGLSL, MeshBVHUniformStruct, FloatVertexAttributeTexture, UIntVertexAttributeTexture } from 'three-mesh-bvh'; function expandGeometry(geometry, size, gutter) { var position = geometry.attributes.position; var normal = geometry.attributes.normal; var tangent = geometry.attributes.tangent; var uv = geometry.attributes.uv; var indices = geometry.index.array; var count = indices.length; var positions = new Float32Array(count * 3); var normals = new Float32Array(count * 3); var tangents = tangent ? new Float32Array(count * 4) : null; var uvs = new Float32Array(count * 2); for (var i = 0; i < count; i += 3) { var a = indices[i], b = indices[i + 1], c = indices[i + 2]; var x0 = uv.getX(a) * size, y0 = uv.getY(a) * size; var x1 = uv.getX(b) * size, y1 = uv.getY(b) * size; var x2 = uv.getX(c) * size, y2 = uv.getY(c) * size; // shift corners away from the opposite edge center var d0 = Math.hypot(x0 * 2 - x1 - x2, y0 * 2 - y1 - y2); var d1 = Math.hypot(x1 * 2 - x0 - x2, y1 * 2 - y0 - y2); var d2 = Math.hypot(x2 * 2 - x0 - x1, y2 * 2 - y0 - y1); var e0 = d0 == 0 ? 0 : gutter / d0; var e1 = d1 == 0 ? 0 : gutter / d1; var e2 = d2 == 0 ? 0 : gutter / d2; for (var j = 0; j < 3; ++j) { // expand each triangle in barycentric space to accommodate extra gutter width var ej = j == 0 ? e0 : j == 1 ? e1 : e2; var w0 = j == 0 ? 1 + ej * 2 : -ej; var w1 = j == 1 ? 1 + ej * 2 : -ej; var w2 = j == 2 ? 1 + ej * 2 : -ej; // interpolate all attributes; we will use UVs to rasterize and positions/normals to trace against the original mesh var v = i + j; positions[v * 3 + 0] = position.getX(a) * w0 + position.getX(b) * w1 + position.getX(c) * w2; positions[v * 3 + 1] = position.getY(a) * w0 + position.getY(b) * w1 + position.getY(c) * w2; positions[v * 3 + 2] = position.getZ(a) * w0 + position.getZ(b) * w1 + position.getZ(c) * w2; normals[v * 3 + 0] = normal.getX(a) * w0 + normal.getX(b) * w1 + normal.getX(c) * w2; normals[v * 3 + 1] = normal.getY(a) * w0 + normal.getY(b) * w1 + normal.getY(c) * w2; normals[v * 3 + 2] = normal.getZ(a) * w0 + normal.getZ(b) * w1 + normal.getZ(c) * w2; if (tangent) { // tangents are only used when normal maps are baked tangents[v * 4 + 0] = tangent.getX(a) * w0 + tangent.getX(b) * w1 + tangent.getX(c) * w2; tangents[v * 4 + 1] = tangent.getY(a) * w0 + tangent.getY(b) * w1 + tangent.getY(c) * w2; tangents[v * 4 + 2] = tangent.getZ(a) * w0 + tangent.getZ(b) * w1 + tangent.getZ(c) * w2; tangents[v * 4 + 3] = tangent.getW(a) * w0 + tangent.getW(b) * w1 + tangent.getW(c) * w2; } uvs[v * 2 + 0] = (x0 * w0 + x1 * w1 + x2 * w2) / size; uvs[v * 2 + 1] = (y0 * w0 + y1 * w1 + y2 * w2) / size; } } var result = new THREE.BufferGeometry(); result.setAttribute('position', new THREE.BufferAttribute(positions, 3)); result.setAttribute('normal', new THREE.BufferAttribute(normals, 3)); result.setAttribute('uv', new THREE.BufferAttribute(uvs, 2)); if (tangent) result.setAttribute('tangent', new THREE.BufferAttribute(tangents, 4)); return result; } function sampleGeometry(geometry, size) { var position = geometry.attributes.position; var normal = geometry.attributes.normal; var indices = geometry.index.array; var vertices = position.count; var triangles = indices.length / 3; var samples = vertices + triangles; // one sample per vertex plus one sample per triangle centroid var positions = new Float32Array(samples * 3); var normals = new Float32Array(samples * 3); var uvs = new Float32Array(samples * 2); for (var i = 0; i < vertices; ++i) { for (var k = 0; k < 3; ++k) { positions[i * 3 + k] = position.getComponent(i, k); normals[i * 3 + k] = normal.getComponent(i, k); } } for (var i = 0; i < triangles; ++i) { var a = indices[i * 3 + 0], b = indices[i * 3 + 1], c = indices[i * 3 + 2]; // normals will be normalized in the shader for (var k = 0; k < 3; ++k) { positions[(vertices + i) * 3 + k] = (position.getComponent(a, k) + position.getComponent(b, k) + position.getComponent(c, k)) / 3; normals[(vertices + i) * 3 + k] = normal.getComponent(a, k) + normal.getComponent(b, k) + normal.getComponent(c, k); } } for (var i = 0; i < samples; ++i) { uvs[i * 2 + 0] = ((i % size) + 0.5) / size; uvs[i * 2 + 1] = (Math.floor(i / size) + 0.5) / size; } var result = new THREE.BufferGeometry(); result.setAttribute('position', new THREE.BufferAttribute(positions, 3)); result.setAttribute('normal', new THREE.BufferAttribute(normals, 3)); result.setAttribute('uv', new THREE.BufferAttribute(uvs, 2)); return result; } function extractColors(data, colors, indices, facets) { var vertices = colors.length / 4; var triangles = indices.length / 3; for (var i = 0; i < vertices; ++i) { if (data[i * 4 + 3] == 0) continue; for (var k = 0; k < 3; ++k) colors[i * 4 + k] = Math.pow(data[i * 4 + k] / 255, 2.2); colors[i * 4 + 3] = 1; } for (var i = 0; i < triangles; ++i) { var offset = (vertices + i) * 4; if (data[offset + 3] == 0) continue; for (var j = 0; j < 3; ++j) { var v = indices[i * 3 + j]; for (var k = 0; k < 3; ++k) colors[v * 4 + k] += Math.pow(data[offset + k] / 255, 2.2); colors[v * 4 + 3]++; } } for (var i = 0; i < colors.length; i += 4) { var weight = colors[i + 3]; if (weight == 0) continue; for (var k = 0; k < 3; ++k) colors[i + k] /= weight; colors[i + 3] = 1; } if (facets) { for (var i = 0; i < triangles; ++i) { var a = indices[i * 3 + 0], b = indices[i * 3 + 1], c = indices[i * 3 + 2]; for (var k = 0; k < 3; ++k) { var avg = (colors[a * 4 + k] + colors[b * 4 + k] + colors[c * 4 + k]) / 3; colors[a * 4 + k] = colors[b * 4 + k] = colors[c * 4 + k] = avg; } } } } var MAP_SIZE = 1024; var MAP_BUDGET = 1 << 30; var MAP_SLOTS = ['map', 'aoMap', 'emissiveMap', 'roughnessMap', 'metalnessMap', 'normalMap']; function createTextureArray(materials) { var textures = []; var layers = new Map(); for (var i = 0; i < materials.length; ++i) { for (var j = 0; j < MAP_SLOTS.length; ++j) { var texture = materials[i][MAP_SLOTS[j]]; // deduplicate images by source UUID to avoid wasting layers for reused images if (texture && texture.image && !layers.has(texture.source)) { layers.set(texture.source, textures.length); textures.push(texture); } } } var depth = Math.max(textures.length, 1); var size = MAP_SIZE; // prevent accidental catastrophic memory use by downscaling texture array until it fits while (size > 4 && depth * size * size * 4 > MAP_BUDGET) size /= 2; // all images need to be resized to the same size because we need all of them in one texture array var canvas = document.createElement('canvas'); canvas.width = canvas.height = size; var context = canvas.getContext('2d', { willReadFrequently: true }); var data = new Uint8Array(size * size * 4 * depth); for (var i = 0; i < textures.length; ++i) { context.clearRect(0, 0, size, size); context.drawImage(textures[i].image, 0, 0, size, size); data.set(context.getImageData(0, 0, size, size).data, i * size * size * 4); } var result = new THREE.DataArrayTexture(data, size, size, depth); result.wrapS = THREE.RepeatWrapping; result.wrapT = THREE.RepeatWrapping; result.minFilter = THREE.LinearFilter; result.magFilter = THREE.LinearFilter; result.needsUpdate = true; return { array: result, layers: layers }; } function createMaterialTexture(materials, layers) { var stride = 24; var count = Math.max(materials.length, 1); var data = new Float32Array(count * stride); function getLayer(material, slot) { var texture = material[slot]; return texture && layers.has(texture.source) ? layers.get(texture.source) : -1; } for (var i = 0; i < materials.length; ++i) { var material = materials[i]; var map = material.map; // simplify UV transform by only taking repeat/offset from main texture data[i * stride + 0] = map ? map.repeat.x : 1; data[i * stride + 1] = map ? map.repeat.y : 1; data[i * stride + 2] = map ? map.offset.x : 0; data[i * stride + 3] = map ? map.offset.y : 0; // layer indices for (var j = 0; j < MAP_SLOTS.length; ++j) { data[i * stride + 4 + j] = getLayer(material, MAP_SLOTS[j]); } // material parameters data[i * stride + 12] = material.color ? material.color.r : 1; data[i * stride + 13] = material.color ? material.color.g : 1; data[i * stride + 14] = material.color ? material.color.b : 1; data[i * stride + 15] = material.aoMapIntensity !== undefined ? material.aoMapIntensity : 1; data[i * stride + 16] = material.emissive ? material.emissive.r * material.emissiveIntensity : 0; data[i * stride + 17] = material.emissive ? material.emissive.g * material.emissiveIntensity : 0; data[i * stride + 18] = material.emissive ? material.emissive.b * material.emissiveIntensity : 0; data[i * stride + 20] = material.metalness !== undefined ? material.metalness : 0; data[i * stride + 21] = material.roughness !== undefined ? material.roughness : 1; data[i * stride + 22] = material.normalScale ? material.normalScale.x : 1; } var texture = new THREE.DataTexture(data, stride / 4, count, THREE.RGBAFormat, THREE.FloatType); texture.needsUpdate = true; return texture; } var bakeShader = new THREE.ShaderMaterial({ glslVersion: THREE.GLSL3, uniforms: { bvh: { value: null }, attrNormal: { value: null }, attrUv: { value: null }, attrColor: { value: null }, attrTangent: { value: null }, attrMaterial: { value: null }, materials: { value: null }, textures: { value: null }, thickness: { value: 0 }, bakeNormals: { value: 0 }, bakeMaterial: { value: 0 }, }, vertexShader: ` attribute vec4 tangent; varying vec3 vPosition; varying vec3 vNormal; varying vec4 vTangent; void main() { vPosition = position; vNormal = normal; vTangent = tangent; gl_Position = vec4(uv * 2.0 - 1.0, 0.0, 1.0); gl_PointSize = 1.0; // for vertex colors } `, fragmentShader: ` precision highp isampler2D; precision highp usampler2D; precision highp sampler2DArray; ${BVHShaderGLSL.common_functions} ${BVHShaderGLSL.bvh_struct_definitions} ${BVHShaderGLSL.bvh_ray_functions} ${BVHShaderGLSL.bvh_distance_functions} uniform BVH bvh; uniform sampler2D attrNormal; uniform sampler2D attrUv; uniform sampler2D attrColor; uniform sampler2D attrTangent; uniform usampler2D attrMaterial; uniform sampler2D materials; uniform sampler2DArray textures; uniform float thickness; uniform float bakeNormals; uniform float bakeMaterial; vec3 fromsrgb(vec3 c) { return pow(c, vec3(2.2)); } vec3 tosrgb(vec3 c) { return pow(c, vec3(1.0 / 2.2)); } varying vec3 vPosition; varying vec3 vNormal; varying vec4 vTangent; layout(location = 0) out vec4 outColor; layout(location = 1) out vec4 outNormal; layout(location = 2) out vec4 outMaterial; void main() { vec3 normal = normalize(vNormal); uvec4 face = uvec4(0u); vec3 faceNormal = vec3(0.0), bary = vec3(0.0), closest = vec3(0.0); float side = 1.0, dist = 0.0; float maxClosest = thickness * 10.0; if (bvhIntersectFirstHit(bvh, vPosition + normal * thickness, -normal, face, faceNormal, bary, side, dist) && dist < thickness * 2.0) { // ray hit } else if (bvhClosestPointToPoint(bvh, vPosition, maxClosest, face, faceNormal, bary, side, closest) < maxClosest) { // closest point found; note, bary is ordered incorrectly in three-mesh-bvh for closest point so we need to swizzle it bary = bary.zxy; } else { discard; } vec2 uv = textureSampleBarycoord(attrUv, bary, face.xyz).xy; int mati = int(uTexelFetch1D(attrMaterial, face.x).r); vec4 md0 = texelFetch(materials, ivec2(0, mati), 0); vec4 md1 = texelFetch(materials, ivec2(1, mati), 0); vec4 md2 = texelFetch(materials, ivec2(2, mati), 0); vec4 md3 = texelFetch(materials, ivec2(3, mati), 0); vec4 md4 = texelFetch(materials, ivec2(4, mati), 0); vec4 md5 = texelFetch(materials, ivec2(5, mati), 0); // md0: uv transform, md1.xyzw + md2.xy: layer indices in MAP_SLOTS order, // md3: base color + ao intensity, md4.xyz: emissive color, md5.xyz: metalness + roughness + normal scale vec2 point = uv * md0.xy + md0.zw; vec3 color = md3.rgb; if (md1.x >= 0.0) color *= fromsrgb(texture(textures, vec3(point, md1.x)).rgb); // vertex colors are linear, and are white when the source mesh or material doesn't use them color *= textureSampleBarycoord(attrColor, bary, face.xyz).rgb; if (md1.y >= 0.0) { float occlusion = texture(textures, vec3(point, md1.y)).r; color *= 1.0 + md3.w * (occlusion - 1.0); } float roughness = md5.y; float metalness = md5.x; if (md1.w >= 0.0) roughness *= texture(textures, vec3(point, md1.w)).g; if (md2.x >= 0.0) metalness *= texture(textures, vec3(point, md2.x)).b; // approximate metalness/roughness impact when material is not baked if (bakeMaterial < 0.5) color *= 1.0 - metalness * (1.0 - roughness * roughness * roughness); vec3 emissive = md4.rgb; if (md1.z >= 0.0) emissive *= fromsrgb(texture(textures, vec3(point, md1.z)).rgb); color += emissive; outColor = vec4(tosrgb(color), 1.0); outNormal = vec4(0.0); outMaterial = vec4(0.0, roughness, metalness, 1.0); if (bakeNormals > 0.5) { vec3 snormal = textureSampleBarycoord(attrNormal, bary, face.xyz).xyz; if (md2.y >= 0.0) { vec4 stangent = textureSampleBarycoord(attrTangent, bary, face.xyz); vec3 bitangent = cross(snormal, stangent.xyz) * (stangent.w < 0.0 ? -1.0 : 1.0); vec3 nmap = texture(textures, vec3(point, md2.y)).xyz * 2.0 - 1.0; nmap.xy *= md5.z; // replace source (smooth) normal with perturbed normal snormal = stangent.xyz * nmap.x + bitangent * nmap.y + snormal * nmap.z; } vec3 axisN = normal; vec3 axisT = vTangent.xyz - axisN * dot(axisN, vTangent.xyz); vec3 axisB = cross(axisN, axisT) * (vTangent.w < 0.0 ? -1.0 : 1.0); vec3 tbn = vec3(dot(snormal, axisT), dot(snormal, axisB), dot(snormal, axisN)); outNormal = vec4(normalize(tbn) * 0.5 + 0.5, 1.0); } } `, depthTest: false, depthWrite: false, side: THREE.DoubleSide, }); var infillShader = new THREE.ShaderMaterial({ glslVersion: THREE.GLSL3, uniforms: { map: { value: null }, }, vertexShader: ` void main() { gl_Position = vec4(vec2(gl_VertexID & 1, gl_VertexID >> 1) * 4.0 - 1.0, 0.0, 1.0); } `, fragmentShader: ` uniform sampler2D map; layout(location = 0) out vec4 result; void main() { ivec2 p = ivec2(gl_FragCoord.xy); vec2 uv = (vec2(p) + 0.5) / vec2(textureSize(map, 0)); // use first texel with non-zero alpha as fallback vec4 c = texelFetch(map, p, 0); for (int l = 1; l <= 10 && c.a < 1e-4; ++l) c = textureLod(map, uv, float(l)); // filtering results in premultiplied value that we need to unpremultiply result = vec4(c.rgb / max(c.a, 1e-4), 1.0); } `, depthTest: false, depthWrite: false, }); function bindCache(cache, thickness, bakeNormals, bakeMaterial) { bakeShader.uniforms.bvh.value = cache.bvh; bakeShader.uniforms.attrNormal.value = cache.attributes.normal; bakeShader.uniforms.attrUv.value = cache.attributes.uv; bakeShader.uniforms.attrColor.value = cache.attributes.color; bakeShader.uniforms.attrTangent.value = cache.attributes.tangent; bakeShader.uniforms.attrMaterial.value = cache.attributes.material; bakeShader.uniforms.materials.value = cache.materials; bakeShader.uniforms.textures.value = cache.textures; bakeShader.uniforms.thickness.value = thickness; bakeShader.uniforms.bakeNormals.value = bakeNormals ? 1 : 0; bakeShader.uniforms.bakeMaterial.value = bakeMaterial ? 1 : 0; } function renderSamples(renderer, object, size, count, infill) { var target = new THREE.WebGLRenderTarget(size, size, { count: count, minFilter: THREE.LinearMipmapNearestFilter, // we use linear filtering with explicit lod for infill depthBuffer: false, generateMipmaps: infill, }); var infilled = []; for (var i = 0; infill && i < count; ++i) { infilled.push(new THREE.WebGLRenderTarget(size, size, { depthBuffer: false })); } renderer.setClearColor(0x000000, 0); renderer.setRenderTarget(target); renderer.render(object, new THREE.OrthographicCamera()); if (infill) { var infillQuad = new THREE.Mesh(new THREE.BufferGeometry(), infillShader); infillQuad.geometry.setDrawRange(0, 3); infillQuad.frustumCulled = false; for (var i = 0; i < count; ++i) { infillShader.uniforms.map.value = target.textures[i]; renderer.setRenderTarget(infilled[i]); renderer.render(infillQuad, new THREE.OrthographicCamera()); } } renderer.setRenderTarget(null); var data = []; for (var i = 0; i < count; ++i) { data.push(new Uint8Array(size * size * 4)); renderer.readRenderTargetPixels(infill ? infilled[i] : target, 0, 0, size, size, data[i], undefined, infill ? 0 : i); } target.dispose(); for (var i = 0; i < infilled.length; ++i) infilled[i].dispose(); return data; } // data comes from render texture, but we need it as DataTexture to be compatible with canvas/GLB export function createTexture(data, size, srgb) { var texture = new THREE.DataTexture(data, size, size, THREE.RGBAFormat); if (srgb) texture.colorSpace = THREE.SRGBColorSpace; texture.magFilter = THREE.LinearFilter; texture.minFilter = THREE.LinearFilter; texture.needsUpdate = true; return texture; } export function buildCache(geometry, materials) { var bvh = new MeshBVH(geometry, { indirect: true }); var bvhgpu = new MeshBVHUniformStruct(); var attrgpu = { normal: new FloatVertexAttributeTexture(), uv: new FloatVertexAttributeTexture(), color: new FloatVertexAttributeTexture(), tangent: new FloatVertexAttributeTexture(), material: new UIntVertexAttributeTexture(), }; bvhgpu.updateFrom(bvh); attrgpu.normal.updateFrom(geometry.attributes.normal); attrgpu.uv.updateFrom(geometry.attributes.uv); attrgpu.color.updateFrom(geometry.attributes.color); attrgpu.tangent.updateFrom(geometry.attributes.tangent); attrgpu.material.updateFrom(geometry.attributes.material); var textures = createTextureArray(materials); var matgpu = createMaterialTexture(materials, textures.layers); return { bvh: bvhgpu, attributes: attrgpu, materials: matgpu, textures: textures.array, dispose: function () { this.bvh.dispose(); this.materials.dispose(); this.textures.dispose(); for (var k in this.attributes) this.attributes[k].dispose(); }, }; } export function transferTexture(renderer, geometry, cache, thickness, size, gutter, infill, bakeNormals, bakeMaterial) { var expanded = expandGeometry(geometry, size, gutter); var mesh = new THREE.Mesh(expanded, bakeShader); mesh.frustumCulled = false; bindCache(cache, thickness, bakeNormals, bakeMaterial); var targets = bakeMaterial ? 3 : bakeNormals ? 2 : 1; var data = renderSamples(renderer, mesh, size, targets, infill); expanded.dispose(); return { map: createTexture(data[0], size, true), normalMap: bakeNormals ? createTexture(data[1], size, false) : null, materialMap: bakeMaterial ? createTexture(data[2], size, false) : null, }; } export function transferColors(renderer, geometry, cache, thickness, facets) { var vertices = geometry.attributes.position.count; var triangles = geometry.index.array.length / 3; var size = Math.ceil(Math.sqrt(vertices + triangles)); var samples = sampleGeometry(geometry, size); var points = new THREE.Points(samples, bakeShader); points.frustumCulled = false; bindCache(cache, thickness, false, false); var data = renderSamples(renderer, points, size, 1)[0]; samples.dispose(); var colors = new Float32Array(vertices * 4); extractColors(data, colors, geometry.index.array, facets); geometry.setAttribute('color', new THREE.BufferAttribute(colors, 4)); }