Fix blend shape error if use quantization (#2031)

* fix: glTF blend shape quatizate compress

* fix: accessor.bufferView is undefined
This commit is contained in:
ChenMo
2024-03-15 17:40:51 +08:00
committed by GitHub
parent 309de9baaf
commit 1dda49d642
6 changed files with 216 additions and 231 deletions

View File

@@ -0,0 +1,60 @@
/**
* @title Animation BlendShape Quantization
* @category Animation
*/
import {
Animator,
Camera,
Color,
DirectLight,
Entity,
GLTFResource,
SkinnedMeshRenderer,
Vector3,
WebGLEngine
} from "@galacean/engine";
import { initScreenshot, updateForE2E } from "./.mockForE2E";
WebGLEngine.create({
canvas: "canvas",
glTF: { meshOpt: { workerCount: 0 } }
}).then(async (engine) => {
engine.canvas.resizeByClientSize();
const scene = engine.sceneManager.activeScene;
const rootNode = scene.createRootEntity();
scene.ambientLight.diffuseSolidColor.set(1, 1, 1, 1);
const directLightEntity = rootNode.createChild("light");
directLightEntity.transform.setPosition(-9, 15, 17);
directLightEntity.transform.lookAt(new Vector3(0, 0, 0));
const directLightComp = directLightEntity.addComponent(DirectLight);
directLightComp.color = new Color(1, 1, 1, 1);
directLightComp.intensity = 1;
// Create camera
const cameraNode = new Entity(engine, "camera_node");
cameraNode.transform.position = new Vector3(0, 0, 30);
const camera = cameraNode.addComponent(Camera);
camera.nearClipPlane = 0.1;
camera.farClipPlane = 1000;
scene.addRootEntity(cameraNode);
cameraNode.transform.lookAt(new Vector3());
engine.resourceManager
.load<GLTFResource>(
"https://mdn.alipayobjects.com/oasis_be/afts/file/A*9eZ0SJBf8ZsAAAAAAAAAAAAADkp5AQ/0312Ani_12FPS_tex.glb"
)
.then((gltf) => {
const gltfEntity = gltf.defaultSceneRoot;
gltf.animations?.forEach((item) => console.log(item.name));
gltfEntity.getComponent(Animator)!;
rootNode.addChild(gltfEntity);
const animator = gltfEntity.getComponentsIncludeChildren(SkinnedMeshRenderer, []);
animator.forEach((item) => {
item.blendShapeWeights[3] = 1.0;
});
updateForE2E(engine);
initScreenshot(engine, camera);
});
});

View File

@@ -10,6 +10,11 @@ export const E2E_CONFIG = {
caseFileName: "animator-blendShape",
threshold: 0.1
},
blendShapeQuantization: {
category: "Animator",
caseFileName: "animator-blendShape-quantization",
threshold: 0.1
},
crossfade: {
category: "Animator",
caseFileName: "animator-crossfade",

View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:7b55012e4939051e85fcb22892b9b615e98dd7e289e9e4d54d7802a94fa63e2d
size 242978

View File

@@ -11,7 +11,7 @@ import {
import { RequestConfig } from "@galacean/engine-core/types/asset/request";
import { Vector2 } from "@galacean/engine-math";
import { GLTFResource } from "./gltf/GLTFResource";
import type { IBufferView } from "./gltf/GLTFSchema";
import type { AccessorComponentType, IBufferView } from "./gltf/GLTFSchema";
import { GLTFUtils } from "./gltf/GLTFUtils";
import { KTX2Loader } from "./ktx2/KTX2Loader";
@@ -88,9 +88,10 @@ export class GLTFContentRestorer extends ContentRestorer<GLTFResource> {
const positionData = this._getBufferData(buffers, position.buffer);
frame.deltaPositions = GLTFUtils.bufferToVector3Array(
positionData,
position.stride,
position.byteOffset,
position.count
position.count,
position.normalized,
position.componentType
);
if (restoreInfo.normal) {
@@ -98,9 +99,10 @@ export class GLTFContentRestorer extends ContentRestorer<GLTFResource> {
const normalData = this._getBufferData(buffers, normal.buffer);
frame.deltaNormals = GLTFUtils.bufferToVector3Array(
normalData,
normal.stride,
normal.byteOffset,
normal.count
normal.count,
normal.normalized,
normal.componentType
);
}
@@ -109,9 +111,10 @@ export class GLTFContentRestorer extends ContentRestorer<GLTFResource> {
const tangentData = this._getBufferData(buffers, tangent.buffer);
frame.deltaTangents = GLTFUtils.bufferToVector3Array(
tangentData,
tangent.stride,
tangent.byteOffset,
tangent.count
tangent.count,
tangent.normalized,
tangent.componentType
);
}
}
@@ -127,8 +130,13 @@ export class GLTFContentRestorer extends ContentRestorer<GLTFResource> {
private _getBufferData(buffers: ArrayBuffer[], restoreInfo: BufferDataRestoreInfo): TypedArray {
const main = restoreInfo.main;
const buffer = buffers[main.bufferIndex];
const data = new main.TypedArray(buffer, main.byteOffset, main.length);
let data: TypedArray;
if (main) {
const buffer = buffers[main.bufferIndex];
data = new main.TypedArray(buffer, main.byteOffset, main.length);
} else {
data = new main.TypedArray(main.length);
}
const sparseCount = restoreInfo.sparseCount;
if (sparseCount) {
@@ -213,7 +221,13 @@ export class BufferDataRestoreInfo {
export class RestoreDataAccessor {
constructor(
public bufferIndex: number,
public TypedArray: new (buffer: ArrayBuffer, byteOffset: number, length?: number) => TypedArray,
public TypedArray:
| Uint8ArrayConstructor
| Int8ArrayConstructor
| Int16ArrayConstructor
| Uint16ArrayConstructor
| Uint32ArrayConstructor
| Float32ArrayConstructor,
public byteOffset: number,
public length: number
) {}
@@ -237,8 +251,9 @@ export class BlendShapeRestoreInfo {
export class BlendShapeDataRestoreInfo {
constructor(
public buffer: BufferDataRestoreInfo,
public stride: number,
public byteOffset: number,
public count: number
public count: number,
public normalized: boolean,
public componentType: AccessorComponentType
) {}
}

View File

@@ -19,7 +19,7 @@ import {
TextureMagFilter,
TextureMinFilter
} from "./GLTFSchema";
import { GLTFParser, GLTFTextureParser } from "./parser";
import { GLTFTextureParser } from "./parser";
import { BufferInfo, GLTFParserContext, GLTFParserType } from "./parser/GLTFParserContext";
/**
@@ -134,135 +134,88 @@ export class GLTFUtils {
accessor: IAccessor
): Promise<BufferInfo> {
const componentType = accessor.componentType;
const bufferViewIndex = accessor.bufferView ?? 0;
const bufferView = bufferViews[bufferViewIndex];
const TypedArray = GLTFUtils.getComponentType(componentType);
const dataElementSize = GLTFUtils.getAccessorTypeSize(accessor.type);
const dataElementBytes = TypedArray.BYTES_PER_ELEMENT;
const elementStride = dataElementSize * dataElementBytes;
const accessorCount = accessor.count;
return context.get<Uint8Array>(GLTFParserType.BufferView, accessor.bufferView).then((bufferViewData) => {
const bufferIndex = bufferView.buffer;
const bufferByteOffset = bufferViewData.byteOffset ?? 0;
const byteOffset = accessor.byteOffset ?? 0;
let promise: Promise<BufferInfo>;
const TypedArray = GLTFUtils.getComponentType(componentType);
const dataElementSize = GLTFUtils.getAccessorTypeSize(accessor.type);
const dataElementBytes = TypedArray.BYTES_PER_ELEMENT;
const elementStride = dataElementSize * dataElementBytes;
const accessorCount = accessor.count;
const bufferStride = bufferView.byteStride;
if (accessor.bufferView !== undefined) {
const bufferViewIndex = accessor.bufferView;
const bufferView = bufferViews[bufferViewIndex];
let bufferInfo: BufferInfo;
// According to the glTF official documentation only byteStride not undefined is allowed
if (bufferStride !== undefined && bufferStride !== elementStride) {
const bufferSlice = Math.floor(byteOffset / bufferStride);
const bufferCacheKey = bufferViewIndex + ":" + componentType + ":" + bufferSlice + ":" + accessorCount;
const accessorBufferCache = context.accessorBufferCache;
bufferInfo = accessorBufferCache[bufferCacheKey];
if (!bufferInfo) {
const offset = bufferByteOffset + bufferSlice * bufferStride;
const count = accessorCount * (bufferStride / dataElementBytes);
promise = context.get<Uint8Array>(GLTFParserType.BufferView, accessor.bufferView).then((bufferViewData) => {
const bufferIndex = bufferView.buffer;
const bufferByteOffset = bufferViewData.byteOffset ?? 0;
const byteOffset = accessor.byteOffset ?? 0;
const bufferStride = bufferView.byteStride;
let bufferInfo: BufferInfo;
// According to the glTF official documentation only byteStride not undefined is allowed
if (bufferStride !== undefined && bufferStride !== elementStride) {
const bufferSlice = Math.floor(byteOffset / bufferStride);
const bufferCacheKey = bufferViewIndex + ":" + componentType + ":" + bufferSlice + ":" + accessorCount;
const accessorBufferCache = context.accessorBufferCache;
bufferInfo = accessorBufferCache[bufferCacheKey];
if (!bufferInfo) {
const offset = bufferByteOffset + bufferSlice * bufferStride;
const count = accessorCount * (bufferStride / dataElementBytes);
const data = new TypedArray(bufferViewData.buffer, offset, count);
accessorBufferCache[bufferCacheKey] = bufferInfo = new BufferInfo(data, true, bufferStride);
bufferInfo.restoreInfo = new BufferDataRestoreInfo(
new RestoreDataAccessor(bufferIndex, TypedArray, offset, count)
);
}
} else {
const offset = bufferByteOffset + byteOffset;
const count = accessorCount * dataElementSize;
const data = new TypedArray(bufferViewData.buffer, offset, count);
accessorBufferCache[bufferCacheKey] = bufferInfo = new BufferInfo(data, true, bufferStride);
bufferInfo = new BufferInfo(data, false, elementStride);
bufferInfo.restoreInfo = new BufferDataRestoreInfo(
new RestoreDataAccessor(bufferIndex, TypedArray, offset, count)
);
}
} else {
const offset = bufferByteOffset + byteOffset;
const count = accessorCount * dataElementSize;
const data = new TypedArray(bufferViewData.buffer, offset, count);
bufferInfo = new BufferInfo(data, false, elementStride);
bufferInfo.restoreInfo = new BufferDataRestoreInfo(
new RestoreDataAccessor(bufferIndex, TypedArray, offset, count)
);
}
if (accessor.sparse) {
return GLTFUtils.processingSparseData(context, accessor, bufferInfo).then(() => bufferInfo);
}
return bufferInfo;
});
}
public static bufferToVector3Array(
data: TypedArray,
byteStride: number,
accessorByteOffset: number,
count: number
): Vector3[] {
const bytesPerElement = data.BYTES_PER_ELEMENT;
const offset = (accessorByteOffset % byteStride) / bytesPerElement;
const stride = byteStride / bytesPerElement;
const vector3s = new Array<Vector3>(count);
for (let i = 0; i < count; i++) {
const index = offset + i * stride;
vector3s[i] = new Vector3(data[index], data[index + 1], data[index + 2]);
}
return vector3s;
}
/**
* @deprecated
* Get accessor data.
*/
static getAccessorData(glTF: IGLTF, accessor: IAccessor, buffers: ArrayBuffer[]): TypedArray {
const bufferViews = glTF.bufferViews;
const bufferView = bufferViews[accessor.bufferView ?? 0];
const arrayBuffer = buffers[bufferView.buffer];
const accessorByteOffset = accessor.hasOwnProperty("byteOffset") ? accessor.byteOffset : 0;
const bufferViewByteOffset = bufferView.hasOwnProperty("byteOffset") ? bufferView.byteOffset : 0;
const byteOffset = accessorByteOffset + bufferViewByteOffset;
const accessorTypeSize = GLTFUtils.getAccessorTypeSize(accessor.type);
const length = accessorTypeSize * accessor.count;
const byteStride = bufferView.byteStride ?? 0;
const arrayType = GLTFUtils.getComponentType(accessor.componentType);
let uint8Array;
if (byteStride) {
const accessorByteSize = accessorTypeSize * arrayType.BYTES_PER_ELEMENT;
uint8Array = new Uint8Array(accessor.count * accessorByteSize);
const originalBufferView = new Uint8Array(arrayBuffer, bufferViewByteOffset, bufferView.byteLength);
for (let i = 0; i < accessor.count; i++) {
for (let j = 0; j < accessorByteSize; j++) {
uint8Array[i * accessorByteSize + j] = originalBufferView[i * byteStride + accessorByteOffset + j];
}
}
return bufferInfo;
});
} else {
uint8Array = new Uint8Array(arrayBuffer.slice(byteOffset, byteOffset + length * arrayType.BYTES_PER_ELEMENT));
}
const typedArray = new arrayType(uint8Array.buffer);
if (accessor.sparse) {
const { count, indices, values } = accessor.sparse;
const indicesBufferView = bufferViews[indices.bufferView];
const valuesBufferView = bufferViews[values.bufferView];
const indicesArrayBuffer = buffers[indicesBufferView.buffer];
const valuesArrayBuffer = buffers[valuesBufferView.buffer];
const indicesByteOffset = (indices.byteOffset ?? 0) + (indicesBufferView.byteOffset ?? 0);
const indicesByteLength = indicesBufferView.byteLength;
const valuesByteOffset = (values.byteOffset ?? 0) + (valuesBufferView.byteOffset ?? 0);
const valuesByteLength = valuesBufferView.byteLength;
const indicesType = GLTFUtils.getComponentType(indices.componentType);
const indicesArray = new indicesType(
indicesArrayBuffer,
indicesByteOffset,
indicesByteLength / indicesType.BYTES_PER_ELEMENT
);
const valuesArray = new arrayType(
valuesArrayBuffer,
valuesByteOffset,
valuesByteLength / arrayType.BYTES_PER_ELEMENT
const count = accessorCount * dataElementSize;
const data = new TypedArray(count);
const bufferInfo = new BufferInfo(data, false, elementStride);
bufferInfo.restoreInfo = new BufferDataRestoreInfo(
new RestoreDataAccessor(undefined, TypedArray, undefined, count)
);
for (let i = 0; i < count; i++) {
const replaceIndex = indicesArray[i];
for (let j = 0; j < accessorTypeSize; j++) {
typedArray[replaceIndex * accessorTypeSize + j] = valuesArray[i * accessorTypeSize + j];
}
}
promise = Promise.resolve(bufferInfo);
}
return typedArray;
return accessor.sparse
? promise.then((bufferInfo) =>
GLTFUtils.processingSparseData(context, accessor, bufferInfo).then(() => bufferInfo)
)
: promise;
}
static bufferToVector3Array(
buffer: TypedArray,
byteOffset: number,
count: number,
normalized: boolean,
componentType: AccessorComponentType
): Vector3[] {
const baseOffset = byteOffset / buffer.BYTES_PER_ELEMENT;
const stride = buffer.length / count;
const vertices = new Array<Vector3>(count);
const factor = normalized ? GLTFUtils.getNormalizedComponentScale(componentType) : 1;
for (let i = 0; i < count; i++) {
const index = baseOffset + i * stride;
vertices[i] = new Vector3(buffer[index] * factor, buffer[index + 1] * factor, buffer[index + 2] * factor);
}
return vertices;
}
static getBufferViewData(bufferView: IBufferView, buffers: ArrayBuffer[]): ArrayBuffer {

View File

@@ -1,12 +1,4 @@
import {
BlendShape,
Buffer,
BufferBindFlag,
BufferUsage,
ModelMesh,
TypedArray,
VertexElement
} from "@galacean/engine-core";
import { BlendShape, Buffer, BufferBindFlag, BufferUsage, ModelMesh, VertexElement } from "@galacean/engine-core";
import { Vector3 } from "@galacean/engine-math";
import {
BlendShapeDataRestoreInfo,
@@ -17,7 +9,7 @@ import {
import type { IAccessor, IGLTF, IMesh, IMeshPrimitive } from "../GLTFSchema";
import { GLTFUtils } from "../GLTFUtils";
import { GLTFParser } from "./GLTFParser";
import { BufferInfo, GLTFParserContext, GLTFParserType, registerGLTFParser } from "./GLTFParserContext";
import { GLTFParserContext, GLTFParserType, registerGLTFParser } from "./GLTFParserContext";
@registerGLTFParser(GLTFParserType.Mesh)
export class GLTFMeshParser extends GLTFParser {
@@ -33,9 +25,6 @@ export class GLTFMeshParser extends GLTFParser {
gltfMesh: IMesh,
gltfPrimitive: IMeshPrimitive,
gltf: IGLTF,
getVertexBufferData: (semantic: string) => TypedArray,
getBlendShapeData: (semantic: string, shapeIndex: number) => Promise<BufferInfo>,
getIndexBufferData: () => Promise<TypedArray>,
keepMeshData: boolean
): Promise<ModelMesh> {
const { accessors } = gltf;
@@ -151,7 +140,7 @@ export class GLTFMeshParser extends GLTFParser {
// BlendShapes
if (targets) {
promises.push(
GLTFMeshParser._createBlendShape(mesh, meshRestoreInfo, gltfMesh, accessors, targets, getBlendShapeData)
GLTFMeshParser._createBlendShape(context, mesh, meshRestoreInfo, gltfMesh, gltfPrimitive, targets)
);
}
@@ -163,98 +152,77 @@ export class GLTFMeshParser extends GLTFParser {
});
}
private static _getBlendShapeData(
context: GLTFParserContext,
glTF: IGLTF,
accessor: IAccessor
): Promise<{ vertices: Vector3[]; restoreInfo: BlendShapeDataRestoreInfo }> {
return GLTFUtils.getAccessorBuffer(context, glTF.bufferViews, accessor).then((bufferInfo) => {
const buffer = bufferInfo.data;
const byteOffset = bufferInfo.interleaved ? (accessor.byteOffset ?? 0) % bufferInfo.stride : 0;
const { count, normalized, componentType } = accessor;
const vertices = GLTFUtils.bufferToVector3Array(buffer, byteOffset, count, normalized, componentType);
const restoreInfo = new BlendShapeDataRestoreInfo(
bufferInfo.restoreInfo,
byteOffset,
count,
normalized,
componentType
);
return { vertices, restoreInfo };
});
}
/**
* @internal
*/
static _createBlendShape(
context: GLTFParserContext,
mesh: ModelMesh,
meshRestoreInfo: ModelMeshRestoreInfo,
glTFMesh: IMesh,
accessors: IAccessor[],
gltfPrimitive: IMeshPrimitive,
glTFTargets: {
[name: string]: number;
}[],
getBlendShapeData: (semantic: string, shapeIndex: number) => Promise<BufferInfo>
}[]
): Promise<void[]> {
const glTF = context.glTF;
const accessors = glTF.accessors;
const blendShapeNames = glTFMesh.extras ? glTFMesh.extras.targetNames : null;
let promises = new Array<Promise<void>>();
for (let i = 0, n = glTFTargets.length; i < n; i++) {
const name = blendShapeNames ? blendShapeNames[i] : `blendShape${i}`;
const targets = gltfPrimitive.targets[i];
const normalTarget = targets["NORMAL"];
const tangentTarget = targets["TANGENT"];
const hasNormal = normalTarget !== undefined;
const hasTangent = tangentTarget !== undefined;
const promise = Promise.all([
getBlendShapeData("POSITION", i),
getBlendShapeData("NORMAL", i),
getBlendShapeData("TANGENT", i)
]).then((infos) => {
const posBufferInfo = infos[0];
const norBufferInfo = infos[1];
const tanBufferInfo = infos[2];
const target = glTFTargets[i];
let posAccessor: IAccessor;
let norAccessor: IAccessor;
let tanAccessor: IAccessor;
let positions: Vector3[] = null;
if (posBufferInfo) {
posAccessor = accessors[target["POSITION"]];
positions = GLTFUtils.bufferToVector3Array(
posBufferInfo.data,
posBufferInfo.stride,
posAccessor.byteOffset ?? 0,
posAccessor.count
);
}
let normals: Vector3[] = null;
if (norBufferInfo) {
norAccessor = accessors[target["NORMAL"]];
normals = GLTFUtils.bufferToVector3Array(
norBufferInfo.data,
norBufferInfo.stride,
norAccessor.byteOffset ?? 0,
norAccessor.count
);
}
let tangents: Vector3[] = null;
if (tanBufferInfo) {
tanAccessor = accessors[target["NORMAL"]];
tangents = GLTFUtils.bufferToVector3Array(
tanBufferInfo.data,
tanBufferInfo.stride,
tanAccessor.byteOffset ?? 0,
tanAccessor.count
);
}
this._getBlendShapeData(context, glTF, accessors[targets["POSITION"]]),
hasNormal ? this._getBlendShapeData(context, glTF, accessors[normalTarget]) : null,
hasTangent ? this._getBlendShapeData(context, glTF, accessors[tangentTarget]) : null
]).then((vertices) => {
const [positionData, normalData, tangentData] = vertices;
const blendShape = new BlendShape(name);
blendShape.addFrame(1.0, positions, normals, tangents);
blendShape.addFrame(
1.0,
positionData.vertices,
hasNormal ? normalData.vertices : null,
hasTangent ? tangentData.vertices : null
);
mesh.addBlendShape(blendShape);
meshRestoreInfo.blendShapes.push(
new BlendShapeRestoreInfo(
blendShape,
new BlendShapeDataRestoreInfo(
posBufferInfo.restoreInfo,
posBufferInfo.stride,
posAccessor.byteOffset ?? 0,
posAccessor.count
),
norBufferInfo
? new BlendShapeDataRestoreInfo(
norBufferInfo.restoreInfo,
norBufferInfo.stride,
norAccessor.byteOffset ?? 0,
norAccessor.count
)
: null,
tanBufferInfo
? new BlendShapeDataRestoreInfo(
tanBufferInfo.restoreInfo,
tanBufferInfo.stride,
tanAccessor.byteOffset ?? 0,
tanAccessor.count
)
: null
positionData.restoreInfo,
hasNormal ? normalData.restoreInfo : null,
hasTangent ? tangentData?.restoreInfo : null
)
);
});
@@ -307,25 +275,6 @@ export class GLTFMeshParser extends GLTFParser {
meshInfo,
gltfPrimitive,
glTF,
(attributeSemantic) => {
return null;
},
(attributeName, shapeIndex) => {
const shapeAccessorIdx = gltfPrimitive.targets[shapeIndex];
const attributeAccessorIdx = shapeAccessorIdx[attributeName];
if (attributeAccessorIdx) {
const accessor = glTF.accessors[attributeAccessorIdx];
return GLTFUtils.getAccessorBuffer(context, context.glTF.bufferViews, accessor);
} else {
return null;
}
},
() => {
const indexAccessor = glTF.accessors[gltfPrimitive.indices];
return context.get<ArrayBuffer>(GLTFParserType.Buffer).then((buffers) => {
return GLTFUtils.getAccessorData(glTF, indexAccessor, buffers);
});
},
context.params.keepMeshData
).then(resolve);
}