From: Luke Lau Date: Sun, 8 Mar 2020 00:29:06 +0000 (+0000) Subject: First pass at blendshapes X-Git-Url: https://git.lukelau.me/?p=opengl.git;a=commitdiff_plain;h=d2a4cfcd292f5bc422a025d43855e5f4f21fb161 First pass at blendshapes --- diff --git a/Makefile b/Makefile index c74b100..cb78344 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,15 @@ all: bin/main CXX_FLAGS := -g --std=c++17 -march=native -Wall -bin/main: model.o material.o image.o skybox.o program.o ik.o main.o util.o +bin/main: model.o material.o image.o skybox.o program.o ik.o main.o util.o ui.o blendshapes.o clang++ $(CXX_FLAGS) $^ \ -I/usr/local/include -L/usr/local/lib \ -lassimp \ + -framework Cocoa \ -framework OpenGL -framework glut -framework CoreGraphics -framework CoreFoundation -framework ImageIO -lglew -o $@ ctags *.cpp +ui.o: ui.mm ui.h + clang++ -g -march=native -Wall -I/usr/local/include -c $< -o $@ %.o: %.cpp clang++ $(CXX_FLAGS) -I/usr/local/include -c $< -o $@ diff --git a/blendshapes.cpp b/blendshapes.cpp new file mode 100644 index 0000000..8611f62 --- /dev/null +++ b/blendshapes.cpp @@ -0,0 +1,55 @@ +#include +#include +#include "blendshapes.hpp" +#include "util.hpp" + +void createBlendshapes(std::vector blendshapes, std::string neutral, Program p, Blendshapes *res) { + + Assimp::Importer importer; + unsigned int ppFlags = aiProcess_Triangulate | aiProcess_CalcTangentSpace | aiProcess_GenNormals; + importer.ReadFile(neutral, ppFlags); + const aiScene *scene = importer.GetOrphanedScene(); + + res->model = new Model(scene, p); + + assert(scene->mNumMeshes == 1); + + const aiMesh *neutralMesh = scene->mMeshes[0]; + res->neutral = std::vector(neutralMesh->mNumVertices); + res->deltas = std::vector>(blendshapes.size()); + for (int i = 0; i < neutralMesh->mNumVertices; i++) + res->neutral[i] = aiVector3DToVec3(neutralMesh->mVertices[i]); + + for (int i = 0; i < blendshapes.size(); i++) { + auto fp = blendshapes[i]; + const aiScene *blendshape = importer.ReadFile(fp, ppFlags); + if (blendshape->mNumMeshes != 1) { + std::cerr << "Too many or too little meshes for the blendshape " << fp << std::endl; + abort(); + } + aiMesh *mesh = blendshape->mMeshes[0]; + std::vector meshDeltas(mesh->mNumVertices); + assert(mesh->mNumVertices == neutralMesh->mNumVertices); + for (int j = 0; j < mesh->mNumVertices; j++) { + glm::vec3 d = aiVector3DToVec3(mesh->mVertices[j]) - aiVector3DToVec3(neutralMesh->mVertices[j]); + meshDeltas[j] = d; + } + res->deltas[i] = meshDeltas; + } + +} + +void interpolateBlendshapes(Blendshapes *b, std::vector weights) { + assert(weights.size() == b->deltas.size()); + const Model::Mesh mesh = b->model->meshes[0]; + + for (int i = 0; i < mesh.ai.mNumVertices; i++) { + glm::vec3 pos = b->neutral[i]; + for (int j = 0; j < b->deltas.size(); j++) { + pos += b->deltas[j][i] * weights[j]; + } + mesh.ai.mVertices[i] = vec3ToaiVector3D(pos); + } + + mesh.updatePosBuffer(); +} diff --git a/blendshapes.hpp b/blendshapes.hpp new file mode 100644 index 0000000..fd3bcc4 --- /dev/null +++ b/blendshapes.hpp @@ -0,0 +1,10 @@ +#include "model.hpp" +struct Blendshapes { + Model *model; + std::vector neutral; + std::vector> deltas; +}; + +void createBlendshapes(std::vector blendshapes, std::string neutral, Program p, Blendshapes *res); + +void interpolateBlendshapes(Blendshapes *b, std::vector amounts); diff --git a/compile_flags.txt b/compile_flags.txt index 7ecd5cf..fe4ccad 100644 --- a/compile_flags.txt +++ b/compile_flags.txt @@ -2,3 +2,6 @@ -std=c++17 -I/usr/local/include/ -DDEBUG_NODES +-Ilibui +-Llibui +-lui diff --git a/main.cpp b/main.cpp index 9a49d4c..776b3d6 100644 --- a/main.cpp +++ b/main.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #ifdef __APPLE__ #include #else @@ -22,6 +23,8 @@ #include "image.hpp" #include "util.hpp" #include "ik.hpp" +#include "blendshapes.hpp" +#include "ui.h" #pragma clang diagnostic ignored "-Wdeprecated-declarations" @@ -41,7 +44,11 @@ glm::vec3 camPos = {0, 0, -5}, camFront = {0, 0, 1}, camUp = {0, 1, 0}; float fov = glm::radians(30.f), znear = 0.01f, zfar = 10000.f; float yaw = 1.57, pitch = 0; -glm::vec3 selectedPos; +Model *targetModel; // The model that the selection is happening on +glm::vec3 closestVertex; +std::vector manipulators; +Blendshapes blendshapes; +float *blendshapeWeights; struct Light { glm::mat4 trans; @@ -54,6 +61,12 @@ bool discoLights = false; int windowWidth = 800, windowHeight = 600; +enum Mode { + Default, + Blendshapes +}; +Mode curMode; + float aspect() { return (float)windowWidth / (float)windowHeight; } @@ -136,9 +149,8 @@ glm::mat4 worldSpaceToModelSpace(aiNode *node, glm::mat4 m) { return res; } -void pickVertex() { - auto res = sceneModel->closestVertex(sceneModel->getRoot(), camPos, selectedPos); - drawBox(glm::translate(glm::mat4(1), res.first), {1, 1, 0.5}); +void highlightVertex() { + drawBox(glm::translate(glm::mat4(1), closestVertex), {1, 1, 0.5}); } void display() { @@ -193,9 +205,17 @@ void display() { } #endif + if (curMode == Default) sceneModel->draw(skyboxes[activeSkybox], d * 1000); - pickVertex(); + if (curMode == Blendshapes) { + highlightVertex(); + + for (auto v: manipulators) + drawBox(glm::translate(glm::mat4(1), v), {0.2, 1, 0}); + + blendshapes.model->draw(skyboxes[activeSkybox], d * 1000); + } for (Light &light: lights) drawLight(light); @@ -228,6 +248,45 @@ void setupLightBuffers(GLuint progId) { glVertexAttribPointer(posLoc, 3, GL_FLOAT, GL_FALSE, 0, 0); } +void weightsChanged(int blendshape, float weight) { + blendshapeWeights[blendshape] = weight; + std::vector weights; + weights.assign(blendshapeWeights, blendshapeWeights + blendshapes.deltas.size()); + interpolateBlendshapes(&blendshapes, weights); +} + +void loadBlendshapes() { + + // get all the obj files + std::vector blends; + const std::string modelDir = "models/high-res-blendshapes/"; + DIR *blendDir = opendir(modelDir.c_str()); + while (dirent *e = readdir(blendDir)) { + if (e->d_type & DT_DIR) continue; + const std::string name(e->d_name); + if (name == "neutral.obj") continue; + blends.push_back(name); + } + closedir(blendDir); + + std::vector blendFps; + for (auto blend: blends) blendFps.push_back(modelDir + blend); + createBlendshapes(blendFps, modelDir + "neutral.obj", *pbrProg, &blendshapes); + targetModel = blendshapes.model; + + size_t numBlends = blends.size(); + blendshapeWeights = new float[numBlends]; + for (int i = 0; i < numBlends; i++) blendshapeWeights[i] = 0; + const char *names[numBlends]; + for (int i = 0; i < numBlends; i++) names[i] = blends[i].c_str(); + createControlWindow(numBlends, names, weightsChanged); + + camPos = { 0, 22, 81 }; + camFront = { 0, 0, -1 }; + camUp = { 0, 1, 0 }; + zfar = 10000; + znear = 0.1f; +} void init() { initUtilProg(); @@ -245,8 +304,11 @@ void init() { pbrProg = new Program("pbrvert.glsl", "pbrfrag.glsl"); glUseProgram(pbrProg->progId); - const std::string scenePath = "models/blendshapeNeutral.glb"; - const aiScene *scene = importer.ReadFile(scenePath, aiProcess_Triangulate | aiProcess_CalcTangentSpace | aiProcess_GenNormals | aiProcess_FlipUVs); + if (curMode == Default) { + const std::string scenePath = "models/cowedboy.glb"; + const aiScene *scene = importer.ReadFile( + scenePath, aiProcess_Triangulate | aiProcess_CalcTangentSpace | + aiProcess_GenNormals | aiProcess_FlipUVs); if (!scene) { std::cerr << importer.GetErrorString() << std::endl; exit(1); @@ -260,7 +322,8 @@ void init() { camPos = {camTrans[3][0], camTrans[3][1], camTrans[3][2]}; - glm::vec3 camLookAt = glm::vec3(cam->mLookAt.x, cam->mLookAt.y, cam->mLookAt.z); + glm::vec3 camLookAt = + glm::vec3(cam->mLookAt.x, cam->mLookAt.y, cam->mLookAt.z); camFront = camLookAt - camPos; camUp = glm::vec3(cam->mUp.x, cam->mUp.y, cam->mUp.z); @@ -273,13 +336,20 @@ void init() { for (int i = 0; i < scene->mNumLights; i++) { aiLight *light = scene->mLights[i]; - glm::mat4 trans; findNodeTrans(scene->mRootNode, light->mName, &trans); - glm::vec3 col = { light->mColorAmbient.r, light->mColorAmbient.g, light->mColorAmbient.b }; + glm::mat4 trans; + findNodeTrans(scene->mRootNode, light->mName, &trans); + glm::vec3 col = {light->mColorAmbient.r, light->mColorAmbient.g, + light->mColorAmbient.b}; Light l = {trans, col}; lights.push_back(l); } sceneModel = new Model(scene, *pbrProg); + } + + if (curMode == Blendshapes) { + loadBlendshapes(); + } glEnable(GL_DEPTH_TEST); glEnable(GL_CULL_FACE); @@ -303,8 +373,10 @@ void keyboardUp(unsigned char key, int x, int y) { keyStates[key] = false; } -/* #define ENABLE_MOVEMENT */ +int mouseX, mouseY; +bool needToCalculateClosestVertex = false; +/* #define ENABLE_MOVEMENT */ void timer(int _) { #ifdef ENABLE_MOVEMENT float xSpeed = 0.f, ySpeed = 0.f, zSpeed = 0.f; @@ -329,6 +401,19 @@ void timer(int _) { camPos.y += ySpeed; camPos.z += zSpeed * sin(yaw) - xSpeed * cos(yaw); #endif + + if (needToCalculateClosestVertex) { + GLint vpArr[4]; glGetIntegerv(GL_VIEWPORT, vpArr); + glm::vec4 viewport(vpArr[0], vpArr[1], vpArr[2], vpArr[3]); + glm::vec3 selectedPos = glm::unProject(glm::vec3(mouseX * 2, viewport[3] - mouseY * 2, 1), // hidpi + viewMat(), + projMat(), + viewport); + + closestVertex = targetModel->closestVertex(targetModel->getRoot(), camPos, selectedPos).first; + needToCalculateClosestVertex = false; + } + glutPostRedisplay(); glutTimerFunc(16, timer, 0); } @@ -365,15 +450,15 @@ void motion(int x, int y) { } #endif - GLint vpArr[4]; glGetIntegerv(GL_VIEWPORT, vpArr); - glm::vec4 viewport(vpArr[0], vpArr[1], vpArr[2], vpArr[3]); - selectedPos = glm::unProject(glm::vec3(x * 2, viewport[3] - y * 2, 1), // hidpi - viewMat(), //view * model mat - projMat(), - viewport); + mouseX = x; mouseY = y; + needToCalculateClosestVertex = true; + } void mouse(int button, int state, int x, int y) { + if (button == GLUT_LEFT_BUTTON && state == GLUT_UP) + manipulators.push_back(closestVertex); + #ifdef ENABLE_MOVEMENT if (button == GLUT_LEFT_BUTTON && state == GLUT_UP) firstMouse = true; @@ -394,6 +479,8 @@ int main(int argc, char** argv) { glewInit(); + // TODO: parse argv + curMode = Blendshapes; init(); glutKeyboardFunc(keyboard); diff --git a/model.cpp b/model.cpp index 4192645..0a1e352 100644 --- a/model.cpp +++ b/model.cpp @@ -5,7 +5,7 @@ #include #include "util.hpp" -Model::Mesh::Mesh(const aiMesh *aiMesh, GLuint progId) : ai(*aiMesh) { +Model::Mesh::Mesh(const aiMesh *aiMesh, GLuint progId) : progId(progId), ai(*aiMesh) { std::vector vertices, normals, tangents, bitangents; std::vector texCoords; @@ -47,13 +47,12 @@ Model::Mesh::Mesh(const aiMesh *aiMesh, GLuint progId) : ai(*aiMesh) { glGenVertexArrays(1, &vao); glBindVertexArray(vao); - GLuint vbos[6]; glGenBuffers(6, vbos); - GLuint vertexVbo = vbos[0], normalVbo = vbos[1], texCoordVbo = vbos[2], indicesVbo = vbos[3]; + GLuint posVbo = vbos[0], normalVbo = vbos[1], texCoordVbo = vbos[2], indicesVbo = vbos[3]; GLuint boneVbo = vbos[4]; GLuint posLoc = glGetAttribLocation(progId, "pos"); - glBindBuffer(GL_ARRAY_BUFFER, vertexVbo); + glBindBuffer(GL_ARRAY_BUFFER, posVbo); glBufferData(GL_ARRAY_BUFFER, vertices.size() * sizeof(glm::vec3), &vertices[0], GL_STATIC_DRAW); glEnableVertexAttribArray(posLoc); glVertexAttribPointer(posLoc, 3, GL_FLOAT, GL_FALSE, 0, 0); @@ -114,6 +113,15 @@ Model::Mesh::Mesh(const aiMesh *aiMesh, GLuint progId) : ai(*aiMesh) { glVertexAttribPointer(boneWeightLoc, 4, GL_FLOAT, GL_FALSE, sizeof(VertBones), (const GLvoid *)sizeof(VertBones::ids)); } +void Model::Mesh::updatePosBuffer() const { + GLuint posLoc = glGetAttribLocation(progId, "pos"); + GLuint posVbo = vbos[0]; + glBindBuffer(GL_ARRAY_BUFFER, posVbo); + glBufferData(GL_ARRAY_BUFFER, ai.mNumVertices * sizeof(aiVector3D), ai.mVertices, GL_STATIC_DRAW); + glEnableVertexAttribArray(posLoc); + glVertexAttribPointer(posLoc, 3, GL_FLOAT, GL_FALSE, 0, 0); +} + Model::Node::Node(aiNode &node, GLuint progId, AnimMap *am, std::set allBones, Node *p): ai(node), parent(p), progId(progId), animMap(am), isBone(allBones.count(std::string(node.mName.C_Str())) > 0) { for (int i = 0; i < node.mNumMeshes; i++) { meshIndices.push_back(node.mMeshes[i]); @@ -407,32 +415,38 @@ bool Model::Node::operator==(const Model::Node &rhs) const { // Returns closest vertex in world space and distance // a and b define the line in 3d space std::pair Model::closestVertex(Model::Node &n, glm::vec3 a, glm::vec3 b, glm::mat4 parentTrans) const { - float shortestDist = FLT_MAX; - glm::vec3 closest; + float closestDist = FLT_MAX; + glm::vec3 closestVert; + for (int i = 0; i < n.ai.mNumMeshes; i++) { int meshIdx = n.ai.mMeshes[i]; const aiMesh &mesh = meshes[meshIdx].ai; for (int j = 0; j < mesh.mNumVertices; j++) { - glm::vec4 vPos = glm::vec4(aiVector3DToMat4(mesh.mVertices[j]), 1); + if (mesh.HasNormals()) { + auto n = aiVector3DToVec3(mesh.mNormals[j]); + if (glm::dot(n, glm::normalize(b - a)) > 0) + continue; + } + glm::vec4 vPos = glm::vec4(aiVector3DToVec3(mesh.mVertices[j]), 1); // Move from model space -> world space vPos = parentTrans * aiMatrixToMat4(n.ai.mTransformation) * vPos; float dist = glm::distance(glm::vec3(vPos), glm::closestPointOnLine(glm::vec3(vPos), a, b)); - if (dist < shortestDist) { - closest = glm::vec3(vPos); - shortestDist = dist; + if (dist < closestDist) { + closestVert = glm::vec3(vPos); + closestDist = dist; } } } for (auto child: n.getChildren()) { - auto res = closestVertex(*child, a, b, parentTrans * aiMatrixToMat4(n.ai.mTransformation)); - if (res.second < shortestDist) { - closest = res.first; - shortestDist = res.second; + auto childRes = closestVertex(*child, a, b, parentTrans * aiMatrixToMat4(n.ai.mTransformation)); + if (childRes.second < closestDist) { + closestVert = childRes.first; + closestDist = childRes.second; } } - return { closest, shortestDist }; + return { closestVert, closestDist }; } diff --git a/model.hpp b/model.hpp index ecc8842..aaf647a 100644 --- a/model.hpp +++ b/model.hpp @@ -32,18 +32,21 @@ class Model { float weights[4] = {1, 0, 0, 0}; }; + public: + Model(std::vector blendshapes, std::string neutral, Program p); + Model(const aiScene *scene, Program p); + void draw(Skybox skybox, const float tick) const; + struct Mesh { Mesh(const aiMesh *aiMesh, GLuint progId); GLuint progId, vao, numIndices; + GLuint vbos[6]; unsigned int materialIndex; BoneMap boneMap; const aiMesh &ai; + void updatePosBuffer() const; }; - public: - Model(const aiScene *scene, Program p); - void draw(Skybox skybox, const float tick) const; - class Node { public: Node(aiNode &aiNode, GLuint progId, AnimMap *animMap, std::set allBones, Node *parent); @@ -78,10 +81,10 @@ class Model { std::pair closestVertex(Model::Node &node, glm::vec3 a, glm::vec3 b, glm::mat4 parentTrans = glm::mat4(1)) const; + std::vector meshes; + private: const Program program; - - std::vector meshes; Node *root; std::vector materials; diff --git a/models/blendshapeNeutral.glb b/models/blendshapeNeutral.glb deleted file mode 100644 index f327724..0000000 Binary files a/models/blendshapeNeutral.glb and /dev/null differ diff --git a/shapes.hpp b/shapes.hpp index a1a42eb..92cc8ad 100644 --- a/shapes.hpp +++ b/shapes.hpp @@ -111,3 +111,8 @@ constexpr glm::vec3 pyramid[18] = { glm::vec3(1, -1, -1), glm::vec3(1, -1, 1) }; + +// TODO: Add a sphere +/* constexpr std::vector sphere() { */ + +/* } */ diff --git a/ui.h b/ui.h new file mode 100644 index 0000000..deb9655 --- /dev/null +++ b/ui.h @@ -0,0 +1 @@ +void createControlWindow(size_t numBlendshapes, const char **names, void(*weightChanged)(int, float)); diff --git a/ui.mm b/ui.mm new file mode 100644 index 0000000..c2a0c6f --- /dev/null +++ b/ui.mm @@ -0,0 +1,50 @@ +#import +#import "ui.h" + +@interface Controller : NSObject +- (instancetype)initWithCallback:(void(*)(int, float))callback; +- (void)sliderMoved:(id)sender; + +@end + +@implementation Controller { + void (*weightChanged)(int, float); +} + +- (instancetype)initWithCallback:(void(*)(int, float))callback { + if (self = [super init]) { + self->weightChanged = callback; + } + return self; +} + +- (void)sliderMoved:(id)sender { + NSSlider *slider = (NSSlider*)sender; + weightChanged([slider tag], [slider floatValue]); +} +@end + +void createControlWindow(size_t numBlendshapes, const char **names, void(*weightChanged)(int, float)) { + Controller *controller = [[Controller alloc] initWithCallback: weightChanged]; + + NSWindowStyleMask mask = NSWindowStyleMaskTitled | NSWindowStyleMaskClosable | NSWindowStyleMaskResizable | NSWindowStyleMaskUtilityWindow; + NSWindow *window = [[NSPanel alloc] initWithContentRect: NSMakeRect(20, 20, 340, 420) + styleMask:mask backing:NSBackingStoreBuffered defer:NO]; + [window makeKeyAndOrderFront: nil]; + [window setTitle: @"Blendshapes"]; + + NSMutableArray *sliders = [NSMutableArray arrayWithCapacity: numBlendshapes]; + + for (int i = 0; i < numBlendshapes; i++) { + NSSlider *slider = [NSSlider sliderWithTarget: controller action:@selector(sliderMoved:)]; + [slider setTag: i]; + NSTextField *label = [NSTextField labelWithString: [NSString stringWithUTF8String: names[i]]]; + NSStackView *stackView = [NSStackView stackViewWithViews: @[label, slider]]; + sliders[i] = stackView; + } + + NSStackView *stackView = [NSStackView stackViewWithViews: sliders]; + [stackView setOrientation: NSUserInterfaceLayoutOrientationVertical]; + [stackView setEdgeInsets: NSEdgeInsetsMake(16, 16, 16, 16)]; + [window setContentView: stackView]; +} diff --git a/util.hpp b/util.hpp index 543d0c6..4ccef53 100644 --- a/util.hpp +++ b/util.hpp @@ -24,6 +24,10 @@ inline aiMatrix4x4 mat4ToaiMatrix(glm::mat4 from) { return to; } -inline glm::vec3 aiVector3DToMat4(aiVector3D from) { +inline glm::vec3 aiVector3DToVec3(aiVector3D from) { + return {from[0], from[1], from[2]}; +} + +inline aiVector3D vec3ToaiVector3D(glm::vec3 from) { return {from[0], from[1], from[2]}; }