blendshapes.cpp

   1 #include "blendshapes.hpp"
   2 #include "util.hpp"
   3 #include <assimp/Importer.hpp>
   4 #include <assimp/postprocess.h>
   5 #include <dirent.h>
   6 #include <eigen3/Eigen/Dense>
   7 #include <istream>
   8
   9 void createBlendshapes(std::string dir, std::vector<std::string> blends,
  10                                            std::string neutral, Program p, BlendshapeModel *res) {
  11
  12         Assimp::Importer importer;
  13         unsigned int ppFlags = aiProcess_Triangulate | aiProcess_CalcTangentSpace |
  14                                                    aiProcess_GenNormals;
  15         importer.ReadFile(dir + "/" + neutral, ppFlags);
  16         const aiScene *scene = importer.GetOrphanedScene();
  17
  18         res->model = new Model(scene, p);
  19
  20         assert(scene->mNumMeshes == 1);
  21
  22         const aiMesh *neutralMesh = scene->mMeshes[0];
  23         res->neutral = std::vector<glm::vec3>(neutralMesh->mNumVertices);
  24
  25         res->blendshapes = std::vector<Blendshape>(blends.size());
  26
  27         for (int i = 0; i < neutralMesh->mNumVertices; i++)
  28                 res->neutral[i] = aiVector3DToVec3(neutralMesh->mVertices[i]);
  29
  30         for (int i = 0; i < blends.size(); i++) {
  31                 auto fp = dir + "/" + blends[i];
  32                 const aiScene *blendshape = importer.ReadFile(fp, ppFlags);
  33                 if (blendshape->mNumMeshes != 1) {
  34                         std::cerr << "Too many or too little meshes for the blendshape "
  35                                           << fp << std::endl;
  36                         abort();
  37                 }
  38                 aiMesh *mesh = blendshape->mMeshes[0];
  39                 std::vector<glm::vec3> meshDeltas(mesh->mNumVertices);
  40                 assert(mesh->mNumVertices == neutralMesh->mNumVertices);
  41                 for (int j = 0; j < mesh->mNumVertices; j++) {
  42                         glm::vec3 d = aiVector3DToVec3(mesh->mVertices[j]) -
  43                                                   aiVector3DToVec3(neutralMesh->mVertices[j]);
  44                         meshDeltas[j] = d;
  45                 }
  46                 res->blendshapes[i] = {blends[i], meshDeltas, 0};
  47         }
  48 }
  49
  50 void loadBlendshapes(std::string dir, Program p, BlendshapeModel *bsModel) {
  51         std::vector<std::string> names;
  52         DIR *blendDir = opendir(dir.c_str());
  53         while (dirent *e = readdir(blendDir)) {
  54                 if (e->d_type & DT_DIR)
  55                         continue;
  56                 const std::string name(e->d_name);
  57                 const std::string ext = ".obj";
  58                 if (name.compare(name.length() - ext.length(), ext.length(), ext) != 0)
  59                         continue;
  60                 if (name == "neutral.obj")
  61                         continue;
  62                 names.push_back(name);
  63         }
  64         closedir(blendDir);
  65
  66         // alphabetically sort blendshapes
  67         std::sort(names.begin(), names.end());
  68
  69         // load animation
  70         std::ifstream animFile(dir + "/animation.txt");
  71         if (animFile.good()) {
  72                 std::vector<float> weights(names.size());
  73                 int i = 0;
  74                 while (animFile >> weights[i]) {
  75                         i++;
  76                         if (i >= names.size()) {
  77                                 i = 0;
  78                                 bsModel->animation.push_back(weights);
  79                         }
  80                 }
  81                 assert(i == 0); // should not end on an arbitrary number
  82         }
  83
  84         createBlendshapes(dir, names, "neutral.obj", p, bsModel);
  85 }
  86
  87 void interpolateBlendshapes(BlendshapeModel *b) {
  88         const Model::Mesh mesh = b->model->meshes[0];
  89
  90         for (int i = 0; i < mesh.ai.mNumVertices; i++) {
  91                 glm::vec3 pos = b->neutral[i];
  92                 for (auto &blendshape : b->blendshapes) {
  93                         pos += blendshape.deltas[i] * blendshape.weight;
  94                 }
  95
  96                 mesh.ai.mVertices[i] = vec3ToaiVector3D(pos);
  97         }
  98
  99         mesh.updatePosBuffer();
 100 }
 101
 102 void solveWeights(BlendshapeModel *b, std::map<VertIdx, glm::vec3> manips) {
 103
 104         int numBlends = b->blendshapes.size();
 105         int numManips = manips.size();
 106
 107         Eigen::VectorXf manipOffset(3 * numManips); // 3 * numManips x 1
 108         {
 109                 int i = 0;
 110                 for (auto &m : manips) {
 111                         auto origPos = b->neutral[m.first.second];
 112                         auto manipOffsetV = m.second - origPos;
 113                         for (int j = 0; j < 3; j++)
 114                                 manipOffset(i * 3 + j) = manipOffsetV[j];
 115                         i++;
 116                 }
 117         }
 118
 119         Eigen::MatrixXf Bbar(3 * numManips, numBlends); // 3 * numManips x numBlends
 120         for (int i = 0; i < numBlends; i++) {
 121                 int j = 0;
 122                 for (auto &m : manips) {
 123                         for (int k = 0; k < 3; k++)
 124                                 Bbar(j * 3 + k, i) =
 125                                         b->blendshapes[i].deltas[m.first.second][k];
 126                         j++;
 127                 }
 128         }
 129
 130         Eigen::VectorXf prevWeights(numBlends); // numBlends x 1
 131         for (int i = 0; i < numBlends; i++)
 132                 prevWeights(i) = b->blendshapes[i].weight;
 133
 134         float alpha = .1, mu = 1;
 135
 136         Eigen::MatrixXf LHS(3 * numManips + numBlends * 2, numBlends);
 137         LHS << Bbar, alpha * Eigen::MatrixXf::Identity(numBlends, numBlends),
 138                 mu * Eigen::MatrixXf::Identity(numBlends, numBlends);
 139
 140         Eigen::VectorXf RHS(3 * numManips + numBlends * 2);
 141         RHS << manipOffset, alpha * prevWeights, Eigen::VectorXf::Zero(numBlends);
 142
 143         Eigen::VectorXf w =
 144                 LHS.jacobiSvd(Eigen::ComputeThinU | Eigen::ComputeFullV).solve(RHS);
 145
 146         for (int i = 0; i < numBlends; i++)
 147                 b->blendshapes[i].weight = w[i];
 148 }
 149
 150 void stepBlendshapeAnim(BlendshapeModel *b) {
 151         for (int i = 0; i < b->blendshapes.size(); i++) {
 152                 b->blendshapes[i].weight = b->animation[b->curFrame][i];
 153         }
 154         b->curFrame++;
 155         b->curFrame %= b->animation.size();
 156 }