347 lines
9.4 KiB
Markdown
347 lines
9.4 KiB
Markdown
|
+++
|
||
|
|
date = "2023-11-04"
|
||
|
|
tags = ["vrm","3d"]
|
||
|
|
title = "3d-modelとcardを連携してみた"
|
||
|
|
slug = "vrm"
|
||
|
|
+++
|
||
|
|
|
||
|
|
https://card.syui.ai/ai
|
||
|
|
|
||
|
|
### 動作環境
|
||
|
|
|
||
|
|
ios17で動作します。ios16では動作しません。
|
||
|
|
|
||
|
|
- [ok] ... ios17.x
|
||
|
|
- [no] ... ios16.x
|
||
|
|
|
||
|
|
`safari`の以下の機能が必要です。
|
||
|
|
|
||
|
|
ios17のデフォルトでは有効になっています。
|
||
|
|
|
||
|
|
- Allow WebGL in Web Workers
|
||
|
|
- GPU Process: Canvas Rendering
|
||
|
|
- GPU Process: DOM Rendering
|
||
|
|
- OffscreenCanvas in Workers
|
||
|
|
- OffscreenCanvas
|
||
|
|
|
||
|
|
### fix motion
|
||
|
|
|
||
|
|
例えば、以下はblenderでポーズを編集している様子なんだけど、プレビューと出力結果が異なります。
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
これは、アニメーションが自動で動作するように設定されているためです。そのままvrmを読み込むとTポーズになりますが、[JLChnToZ/vrm-dance-viewer](https://github.com/JLChnToZ/vrm-dance-viewer)は設定で手を下げて固定します。
|
||
|
|
|
||
|
|
```ts:src/worker/vrm-idle-helper.ts
|
||
|
|
// https://github.com/JLChnToZ/vrm-dance-viewer
|
||
|
|
const LERP_SCALE = 6;
|
||
|
|
|
||
|
|
if (node) node.setRotationFromQuaternion(
|
||
|
|
rotation3
|
||
|
|
.setFromRotationMatrix(node.matrix)
|
||
|
|
// ここがポーズをおかしくする要因
|
||
|
|
// コメント化すると元通り。ただvrmを自然なポーズに固定する必要がでてくる
|
||
|
|
//.slerp(finalRotation, Math.min(deltaTime * LERP_SCALE, 1)),
|
||
|
|
);
|
||
|
|
```
|
||
|
|
|
||
|
|
また、blinkを設定すると、顔を標準位置から移動するとおかしくなります。これは`three-vrm`の古いバージョンのバグです。
|
||
|
|
|
||
|
|
```ts:src/worker/vrm-idle-helper.ts
|
||
|
|
function updateEyeBlink(model: VRM, deltaTime: number) {
|
||
|
|
//if (!model.blendShapeProxy) return;
|
||
|
|
//let v = blinkDelays.get(model);
|
||
|
|
//if (v == null || v < -BLINK_DURATION)
|
||
|
|
// v = MathUtils.randFloat(MIN_BLINK_DELAY, MAX_BLINK_DEALY);
|
||
|
|
//else
|
||
|
|
// v -= deltaTime;
|
||
|
|
//blinkDelays.set(model, v);
|
||
|
|
//model.blendShapeProxy.setValue(
|
||
|
|
// VRMSchema.BlendShapePresetName.Blink,
|
||
|
|
// v > LOOK_CAMERA_THRESHOLD ? 0 : MathUtils.pingpong(-v, BLINK_DURATION / 2) * 2 / BLINK_DURATION,
|
||
|
|
//);
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
これをどう自然に動かしていけばいいのか悩み中。全部のvrmをデフォルトのTポーズから変更するのもあんまり良くない。
|
||
|
|
|
||
|
|
`three.js`の他に`babylon.js`というものもあるらしい。こちらの[virtual-cast/babylon-vrm-loader](https://github.com/virtual-cast/babylon-vrm-loader)でvrmを読み込めます。
|
||
|
|
|
||
|
|
### add effect
|
||
|
|
|
||
|
|
例えば、model/motionを読み込んだ瞬間に紙吹雪が舞う演出を追加してみます。
|
||
|
|
|
||
|
|
|
||
|
|
```js:src/worker/scene/lights.ts
|
||
|
|
// 空中に紙吹雪
|
||
|
|
function tick_sky() {
|
||
|
|
let s_rot = 0;
|
||
|
|
let s_xp = 10;
|
||
|
|
let s_yp = 10;
|
||
|
|
let s_zp = 10;
|
||
|
|
const s_length = 5000;
|
||
|
|
const s_plane_scale = 0.01;
|
||
|
|
const s_plane = [];
|
||
|
|
for(let i=0; i<s_length; i++){
|
||
|
|
var color = "0x" + Math.floor(Math.random() * 16777215).toString(16);
|
||
|
|
let geometry = new THREE.PlaneGeometry( s_plane_scale, s_plane_scale );
|
||
|
|
var material = new THREE.MeshBasicMaterial({
|
||
|
|
color: Number(color),
|
||
|
|
opacity: 0.8,
|
||
|
|
transparent: true,
|
||
|
|
side: THREE.DoubleSide
|
||
|
|
});
|
||
|
|
s_plane[i] = new THREE.Mesh( geometry, material );
|
||
|
|
s_plane[i].position.x = s_xp * (Math.random() - 0.5);
|
||
|
|
s_plane[i].position.y = s_yp * (Math.random() - 0.5);
|
||
|
|
s_plane[i].position.z = s_zp * (Math.random() - 0.5);
|
||
|
|
scene.add(s_plane[i]);
|
||
|
|
}
|
||
|
|
return s_plane;
|
||
|
|
}
|
||
|
|
|
||
|
|
// 紙吹雪を時間経過で消す処理
|
||
|
|
function tick_sky_remove(){
|
||
|
|
const s_length = 5000;
|
||
|
|
var s_plane = tick_sky();
|
||
|
|
setTimeout(() => {
|
||
|
|
for(let i=0; i<s_length; i++){
|
||
|
|
scene.remove(s_plane[i]);
|
||
|
|
}
|
||
|
|
}, 7000);
|
||
|
|
}
|
||
|
|
|
||
|
|
// サービス名を追加
|
||
|
|
export function toggleTickSky() {
|
||
|
|
if (!navigator.userAgent.match(/iPhone|iPod|iPad|Android.+Mobile/)) {
|
||
|
|
tick_sky_remove();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// ホストへ追加
|
||
|
|
WorkerMessageService.host.on({ setLights, toggleLights, toggleTickSky });
|
||
|
|
```
|
||
|
|
|
||
|
|
```js:src/host/index.ts
|
||
|
|
// サービス登録
|
||
|
|
export function toggleTickSky() {
|
||
|
|
return void workerService.trigger('toggleTickSky');
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
```js:src/main.ts
|
||
|
|
import { toggleTickSky } from './host';
|
||
|
|
|
||
|
|
const el = document.querySelector('#btn-models') as HTMLInputElement | null;
|
||
|
|
if(el != null) {
|
||
|
|
// ボタンを押したときの動作
|
||
|
|
el.addEventListener('click', function(){
|
||
|
|
// サービス:紙吹雪の演出
|
||
|
|
toggleTickSky();
|
||
|
|
// サービス:ライトのon/off
|
||
|
|
toggleLights();
|
||
|
|
});
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
```html:src/index.html
|
||
|
|
<button id='btn-models'>test</button>
|
||
|
|
```
|
||
|
|
|
||
|
|
このようにするとmodel/motionにeffectを追加できます。色々な`scene`を作っていく予定。
|
||
|
|
|
||
|
|
<video controls style="width:100%;"><source src="/ai/video/ai_vrm_0002.mp4"></video>
|
||
|
|
|
||
|
|
### add audio
|
||
|
|
|
||
|
|
```js:src/main.ts
|
||
|
|
import * as THREE from 'three';
|
||
|
|
export const camera = new THREE.PerspectiveCamera(75, 1, 0.1, 1000);
|
||
|
|
const listener = new THREE.AudioListener();
|
||
|
|
camera.add( listener );
|
||
|
|
const sound = new THREE.Audio( listener );
|
||
|
|
const audioLoader = new THREE.AudioLoader();
|
||
|
|
|
||
|
|
function audio_sword() {
|
||
|
|
audioLoader.load( './audio/sword.mp3', function( buffer ) {
|
||
|
|
sound.setBuffer( buffer );
|
||
|
|
sound.setLoop( false );
|
||
|
|
sound.setVolume( 0.2 );
|
||
|
|
sound.play();
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
audio_sword();
|
||
|
|
```
|
||
|
|
|
||
|
|
### add floor
|
||
|
|
|
||
|
|
先程の応用でランダムで動き続ける背景を設定してみます。
|
||
|
|
|
||
|
|
```js:src/worker/scene/lights.ts
|
||
|
|
function tick() {
|
||
|
|
let s_rot = 0;
|
||
|
|
let s_xp = 30;
|
||
|
|
// 0にして床のみ設定する
|
||
|
|
let s_yp = 0;
|
||
|
|
let s_zp = 30;
|
||
|
|
const s_length = 15000;
|
||
|
|
// 大きさもランダムにする
|
||
|
|
const s_plane_scale = Math.floor(Math.random() * 0.09) + 0.01;
|
||
|
|
const s_plane = [];
|
||
|
|
for(let i=0; i<s_length; i++){
|
||
|
|
var color = "0x" + Math.floor(Math.random() * 16777215).toString(16);
|
||
|
|
let geometry = new THREE.PlaneGeometry( s_plane_scale, s_plane_scale );
|
||
|
|
var material = new THREE.MeshBasicMaterial({
|
||
|
|
color: Number(color),
|
||
|
|
opacity: 0.8,
|
||
|
|
transparent: true,
|
||
|
|
side: THREE.DoubleSide
|
||
|
|
});
|
||
|
|
s_plane[i] = new THREE.Mesh( geometry, material );
|
||
|
|
// 向きをランダムに変える
|
||
|
|
s_plane[i].rotation.x = Math.PI / 2 * Math.random();
|
||
|
|
s_plane[i].position.x = s_xp * (Math.random() - 0.5);
|
||
|
|
s_plane[i].position.y = s_yp * (Math.random() - 0.5);
|
||
|
|
s_plane[i].position.z = s_zp * (Math.random() - 0.5);
|
||
|
|
scene.add(s_plane[i]);
|
||
|
|
}
|
||
|
|
return s_plane;
|
||
|
|
}
|
||
|
|
|
||
|
|
function tick_remove(){
|
||
|
|
const s_length = 15000;
|
||
|
|
var s_plane = tick();
|
||
|
|
setTimeout(() => {
|
||
|
|
for(let i=0; i<s_length; i++){
|
||
|
|
scene.remove(s_plane[i]);
|
||
|
|
}
|
||
|
|
}, 2000);
|
||
|
|
}
|
||
|
|
|
||
|
|
// これで床のキラキラが更新を続ける
|
||
|
|
export function toggleTick() {
|
||
|
|
const s = tick();
|
||
|
|
tick_remove();
|
||
|
|
var num = Math.floor(Math.random() * 1990) + 1790;
|
||
|
|
for(let i=0; i<100; i++){
|
||
|
|
var num = Math.floor(Math.random() * 1990) + 1790;
|
||
|
|
setTimeout(() => { tick_remove(); }, num * i);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
### add hdr
|
||
|
|
|
||
|
|
hdr画像を設定します。ただし、backgroundはつけないでください。床はつけてもいいです。
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
```js
|
||
|
|
//背景をHDR
|
||
|
|
import * as THREE from 'three';
|
||
|
|
import { RGBELoader } from "three/examples/jsm/loaders/RGBELoader";
|
||
|
|
let uk_0 = "/img/t.hdr"
|
||
|
|
|
||
|
|
new RGBELoader().load(uk_0, function (texture) {
|
||
|
|
texture.mapping = THREE.EquirectangularReflectionMapping;
|
||
|
|
scene.background = texture;
|
||
|
|
//scene.environment = texture;
|
||
|
|
});
|
||
|
|
|
||
|
|
// scene.background.set(bgColor);
|
||
|
|
```
|
||
|
|
|
||
|
|
hdr : https://polyhaven.com/hdris
|
||
|
|
|
||
|
|
ボタンを押すと場所を移動するように設定するにはこんな感じ。
|
||
|
|
|
||
|
|
```js:src/worker/scene/lights.ts
|
||
|
|
const bgColor = new Color(0x000000);
|
||
|
|
|
||
|
|
function getTels(){
|
||
|
|
let hdr = "/img/syferfontein_0d_clear_puresky_4k.hdr";
|
||
|
|
new RGBELoader().load(hdr, function (texture) {
|
||
|
|
texture.mapping = THREE.EquirectangularReflectionMapping;
|
||
|
|
scene.background = texture;
|
||
|
|
//scene.environment = texture;
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
function floor_default(){
|
||
|
|
const floor = new Mesh(
|
||
|
|
new PlaneBufferGeometry(100, 100),
|
||
|
|
new MeshLambertMaterial({
|
||
|
|
color: 0x999999,
|
||
|
|
depthWrite: true,
|
||
|
|
})
|
||
|
|
);
|
||
|
|
floor.position.y = -0.5;
|
||
|
|
floor.rotation.x = -Math.PI / 2;
|
||
|
|
//const { y } = floor.position;
|
||
|
|
//floor.position.set(0, 0, 0);
|
||
|
|
scene.add(floor);
|
||
|
|
return floor;
|
||
|
|
}
|
||
|
|
|
||
|
|
function floor_grid(){
|
||
|
|
const grid = new GridHelper(50, 100, 0xAAAAAA, 0xAAAAAA);
|
||
|
|
scene.add(grid);
|
||
|
|
grid.position.set(Math.round(0), 0, Math.round(0));
|
||
|
|
return grid;
|
||
|
|
}
|
||
|
|
|
||
|
|
function floor_bg(){
|
||
|
|
const bgColor = new Color(0xffffff);
|
||
|
|
scene.background = new Color(bgColor);
|
||
|
|
scene.fog = new Fog(bgColor, 3, 10);
|
||
|
|
scene.fog?.color.set(bgColor);
|
||
|
|
if (scene.background instanceof Color)
|
||
|
|
scene.background.set(bgColor);
|
||
|
|
else
|
||
|
|
scene.background = bgColor.clone();
|
||
|
|
}
|
||
|
|
|
||
|
|
export const fl_de = floor_default();
|
||
|
|
export const fl_gr = floor_grid();
|
||
|
|
export const fl_bg = floor_bg();
|
||
|
|
|
||
|
|
function floor_default_remove(int: number){
|
||
|
|
if (int == 1){
|
||
|
|
scene.remove(fl_de);
|
||
|
|
scene.remove(fl_gr);
|
||
|
|
getTels();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
floor_default_remove(0);
|
||
|
|
|
||
|
|
export function toggleTel(int: number) {
|
||
|
|
floor_default_remove(int)
|
||
|
|
}
|
||
|
|
|
||
|
|
WorkerMessageService.host.on({ toggleTel });
|
||
|
|
```
|
||
|
|
|
||
|
|
```js:src/host/index.ts
|
||
|
|
export function toggleTel(int: number) {
|
||
|
|
return void workerService.trigger('toggleTel', [int]);
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
```js:src/main.ts
|
||
|
|
import { toggleTel } from './host';
|
||
|
|
const el_tel = document.querySelector('#btn-models_tel') as HTMLInputElement | null;
|
||
|
|
if(el_tel != null) {
|
||
|
|
el_tel.addEventListener('click', (e:Event) => toggleTel(1));
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
### link
|
||
|
|
|
||
|
|
- https://zenn.dev/ikkou/articles/fdb344a713cdf0/
|
||
|
|
|
||
|
|
- https://blog.virtualcast.jp/blog/2019/05/oss-browser-vrm-vci-viewer/
|