-
Notifications
You must be signed in to change notification settings - Fork 19
Filtering feature #36
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
8071d19
2aee370
0c76369
c971a84
6f4446c
0d04071
298ed62
546ae82
d97349b
d7f738a
a835960
a987c47
753a833
dd00af0
40fbe23
a81c6f6
a1f545b
727bc28
e7e7f24
42347a8
86fb0a9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -54,6 +54,12 @@ struct arController { | |
KpmHandle* kpmHandle; | ||
AR2HandleT* ar2Handle; | ||
|
||
#if WITH_FILTERING | ||
ARFilterTransMatInfo *ftmi; | ||
ARdouble filterCutoffFrequency = 60.0; | ||
ARdouble filterSampleRate = 120.0; | ||
#endif | ||
|
||
int detectedPage = -2; // -2 Tracking not inited, -1 tracking inited OK, >= 0 tracking online on page. | ||
|
||
int surfaceSetCount = 0; // Running NFT marker id | ||
|
@@ -96,6 +102,22 @@ extern "C" { | |
NFT API bindings | ||
*/ | ||
|
||
void matrixLerp(ARdouble src[3][4], ARdouble dst[3][4], float interpolationFactor) { | ||
for (int i=0; i<3; i++) { | ||
for (int j=0; j<4; j++) { | ||
dst[i][j] = (1 - interpolationFactor) * src[i][j] + interpolationFactor * dst[i][j]; | ||
} | ||
} | ||
} | ||
|
||
void matrixLerp2(ARdouble src[3][4], ARdouble dst[3][4], float interpolationFactor) { | ||
for (int i=0; i<3; i++) { | ||
for (int j=0; j<4; j++) { | ||
dst[i][j] = dst[i][j] + (src[i][j] - dst[i][j]) / interpolationFactor; | ||
} | ||
} | ||
} | ||
|
||
int getNFTMarkerInfo(int id, int markerIndex) { | ||
if (arControllers.find(id) == arControllers.end()) { return ARCONTROLLER_NOT_FOUND; } | ||
arController *arc = &(arControllers[id]); | ||
|
@@ -108,10 +130,22 @@ extern "C" { | |
int kpmResultNum = -1; | ||
|
||
float trans[3][4]; | ||
|
||
#if WITH_FILTERING | ||
ARdouble transF[3][4]; | ||
ARdouble transFLerp[3][4]; | ||
memset( transFLerp, 0, 3 * 4 * sizeof(ARdouble) ); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. maybe this is not needed but not sure of this. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It is needed, without it the interpolation give weird results. |
||
#endif | ||
|
||
float err = -1; | ||
if (arc->detectedPage == -2) { | ||
kpmMatching( arc->kpmHandle, arc->videoLuma ); | ||
kpmGetResult( arc->kpmHandle, &kpmResult, &kpmResultNum ); | ||
|
||
#if WITH_FILTERING | ||
arc->ftmi = arFilterTransMatInit(arc->filterSampleRate, arc->filterCutoffFrequency); | ||
#endif | ||
|
||
int i, j, k; | ||
int flag = -1; | ||
for( i = 0; i < kpmResultNum; i++ ) { | ||
|
@@ -139,6 +173,28 @@ extern "C" { | |
|
||
if (arc->detectedPage >= 0) { | ||
int trackResult = ar2TrackingMod(arc->ar2Handle, arc->surfaceSet[arc->detectedPage], arc->videoFrame, trans, &err); | ||
|
||
#if WITH_FILTERING | ||
for (int j = 0; j < 3; j++) { | ||
for (int k = 0; k < 4; k++) { | ||
transF[j][k] = trans[j][k]; | ||
} | ||
} | ||
|
||
bool reset; | ||
if (trackResult < 0) { | ||
reset = 1; | ||
} else { | ||
reset = 0; | ||
} | ||
|
||
if (arFilterTransMat(arc->ftmi, transF, reset) < 0) { | ||
ARLOGe("arFilterTransMat error with marker %d.\n", markerIndex); | ||
} | ||
|
||
matrixLerp2(transF, transFLerp, 0.95); | ||
#endif | ||
|
||
if( trackResult < 0 ) { | ||
ARLOGi("Tracking lost. %d\n", trackResult); | ||
arc->detectedPage = -2; | ||
|
@@ -179,6 +235,25 @@ extern "C" { | |
markerIndex, | ||
err, | ||
|
||
#if WITH_FILTERING | ||
|
||
transFLerp[0][0], | ||
transFLerp[0][1], | ||
transFLerp[0][2], | ||
transFLerp[0][3], | ||
|
||
transFLerp[1][0], | ||
transFLerp[1][1], | ||
transFLerp[1][2], | ||
transFLerp[1][3], | ||
|
||
transFLerp[2][0], | ||
transFLerp[2][1], | ||
transFLerp[2][2], | ||
transFLerp[2][3] | ||
|
||
#else | ||
|
||
trans[0][0], | ||
trans[0][1], | ||
trans[0][2], | ||
|
@@ -193,6 +268,8 @@ extern "C" { | |
trans[2][1], | ||
trans[2][2], | ||
trans[2][3] | ||
|
||
#endif | ||
); | ||
} else { | ||
EM_ASM_({ | ||
|
@@ -347,6 +424,13 @@ extern "C" { | |
arPattDetach(arc->arhandle); | ||
arDeleteHandle(arc->arhandle); | ||
arc->arhandle = NULL; | ||
|
||
#if WITH_FILTERING | ||
if (arc->ftmi) { | ||
arFilterTransMatFinal(arc->ftmi); | ||
arc->ftmi = NULL; | ||
} | ||
#endif | ||
} | ||
if (arc->ar3DHandle != NULL) { | ||
ar3DDeleteHandle(&(arc->ar3DHandle)); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
#pragma once | ||
|
||
// This implementation "inspired" by a questionable interpretation of a Kalman Filter. | ||
template<typename PointT> | ||
class NaiveFilter | ||
{ | ||
public: | ||
NaiveFilter(PointT initialEstimate, double lambda) | ||
: m_lastEstimate{ initialEstimate } | ||
, m_lambda{ lambda } | ||
{} | ||
|
||
void Update(PointT& measurement, double deltaT) | ||
{ | ||
auto measurementDelta = measurement - m_lastEstimate; | ||
double measurementError = measurementDelta.dot(measurementDelta); | ||
|
||
double gain = (m_lastEstimateError + EPSILON) / (m_lastEstimateError + measurementError + EPSILON); | ||
|
||
m_lastEstimate += gain * measurementDelta; | ||
m_lastEstimateError = std::max((1.f - gain) * m_lastEstimateError, m_lambda * deltaT * measurementError); | ||
|
||
measurement = m_lastEstimate; | ||
} | ||
|
||
private: | ||
static constexpr double EPSILON = 0.00001; // To prevent gain from becoming NaN. | ||
const double m_lambda{}; | ||
|
||
PointT m_lastEstimate{}; | ||
double m_lastEstimateError{ 10.f }; | ||
}; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,140 @@ | ||
<!DOCTYPE html> | ||
<html> | ||
<head> | ||
<meta charset="utf-8"> | ||
<title>NFT marker example with a WebWorker, WASM and with Three.js</title> | ||
<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=0.5, maximum-scale=1"> | ||
<link rel="stylesheet" href="../css/video-style.css"> | ||
</head> | ||
<body class="loading"> | ||
<div id="loading" > | ||
<img src="../Data/logo.gif"/> | ||
<span class="loading-text">Loading, please wait</span> | ||
</div> | ||
<!-- | ||
================== | ||
STATS | ||
================== | ||
--> | ||
|
||
<div id="stats" class="ui stats"> | ||
|
||
<div id="stats1" class="stats-item"> | ||
<p class="stats-item-title"> | ||
Main | ||
</p> | ||
</div> | ||
|
||
<div id="stats2" class="stats-item"> | ||
<p class="stats-item-title"> | ||
Worker | ||
</p> | ||
</div> | ||
|
||
</div> | ||
|
||
<!-- | ||
================== | ||
CAMERA VIDEO & CANVAS | ||
================== | ||
--> | ||
|
||
<div id="app"> | ||
<video | ||
loop | ||
autoplay | ||
muted | ||
playsinline | ||
id="video"> | ||
</video> | ||
|
||
<canvas id="canvas"></canvas> | ||
</div> | ||
|
||
<a | ||
href="https://raw.githubusercontent.com/artoolkit/artoolkit5/master/doc/Marker%20images/pinball.jpg" | ||
class="ui marker" | ||
target="_blank"> | ||
🖼 Marker Image | ||
</a> | ||
|
||
<script src="../js/third_party/three.js/stats.min.js"></script> | ||
<script src="../js/third_party/three.js/three.min.js"></script> | ||
<script src="threejs_wasm_filter_worker.js"></script> | ||
|
||
<script> | ||
|
||
/** | ||
* STATS | ||
*/ | ||
var statsMain = new Stats(); | ||
statsMain.showPanel( 0 ); // 0: fps, 1: ms, 2: mb, 3+: custom | ||
document.getElementById( 'stats1' ).appendChild( statsMain.dom ); | ||
|
||
var statsWorker = new Stats(); | ||
statsWorker.showPanel( 0 ); // 0: fps, 1: ms, 2: mb, 3+: custom | ||
document.getElementById( 'stats2' ).appendChild( statsWorker.dom ); | ||
|
||
/** | ||
* APP / ELEMENTS | ||
*/ | ||
var container = document.getElementById( 'app' ); | ||
var video = document.getElementById( 'video' ); | ||
var canvas = document.getElementById( 'canvas' ); | ||
|
||
/** | ||
* APP / VIDEO STREAM | ||
*/ | ||
|
||
if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) { | ||
var hint = { | ||
audio: false, | ||
video: true | ||
}; | ||
if( window.innerWidth < 800 ) { | ||
var width = ( window.innerWidth < window.innerHeight ) ? 240 : 360; | ||
var height = ( window.innerWidth < window.innerHeight ) ? 360 : 240; | ||
|
||
var aspectRatio = window.innerWidth / window.innerHeight; | ||
|
||
console.log( width, height ); | ||
|
||
hint = { | ||
audio: false, | ||
video: { | ||
facingMode: 'environment', | ||
width: { min: width, max: width } | ||
}, | ||
}; | ||
|
||
console.log( hint ); | ||
} | ||
|
||
navigator.mediaDevices.getUserMedia( hint ).then( function( stream ) { | ||
video.srcObject = stream; | ||
video.addEventListener( 'loadedmetadata', function() { | ||
video.play(); | ||
|
||
console.log( 'video', video, video.videoWidth, video.videoHeight ); | ||
|
||
start( | ||
container, | ||
markers['pinball'], | ||
video, | ||
video.videoWidth, | ||
video.videoHeight, | ||
canvas, | ||
function() { | ||
statsMain.update() | ||
}, | ||
function() { | ||
statsWorker.update(); | ||
}, | ||
null | ||
); | ||
} ); | ||
} ); | ||
} | ||
</script> | ||
</body> | ||
</html> |
Uh oh!
There was an error while loading. Please reload this page.