Skip to content

Commit c7ee498

Browse files
triniwiztriniwiz
committed
feat: still image detection, pause & torch
1 parent a39ac7e commit c7ee498

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+1180
-42
lines changed

‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
- [@nativescript/mlkit-image-labeling](packages/mlkit-image-labeling/README.md)
66
- [@nativescript/mlkit-object-detection](packages/mlkit-object-detection/README.md)
77
- [@nativescript/mlkit-pose-detection](packages/mlkit-pose-detection/README.md)
8+
- [@nativescript/mlkit-selfie-segmentation](packages/mlkit-selfie-segmentation/README.md)
89
- [@nativescript/mlkit-text-recognition](packages/mlkit-text-recognition/README.md)
910

1011
# How to use?

‎apps/demo-angular/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
"@nativescript/mlkit-image-labeling": "file:../../dist/packages/mlkit-image-labeling",
1010
"@nativescript/mlkit-object-detection": "file:../../dist/packages/mlkit-object-detection",
1111
"@nativescript/mlkit-digital-ink-recognition": "file:../../dist/packages/mlkit-digital-ink-recognition",
12-
"@nativescript/mlkit-pose-detection": "file:../../dist/packages/mlkit-pose-detection"
12+
"@nativescript/mlkit-pose-detection": "file:../../dist/packages/mlkit-pose-detection",
13+
"@nativescript/mlkit-selfie-segmentation": "file:../../dist/packages/mlkit-selfie-segmentation"
1314
},
1415
"devDependencies": {
1516
"@nativescript/android": "~8.1.1",

‎apps/demo-angular/src/app-routing.module.ts

+1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ const routes: Routes = [
1414
{ path: 'mlkit-image-labeling', loadChildren: () => import('./plugin-demos/mlkit-image-labeling.module').then(m => m.MlkitImageLabelingModule) },
1515
{ path: 'mlkit-object-detection', loadChildren: () => import('./plugin-demos/mlkit-object-detection.module').then(m => m.MlkitObjectDetectionModule) },
1616
{ path: 'mlkit-pose-detection', loadChildren: () => import('./plugin-demos/mlkit-pose-detection.module').then(m => m.MlkitPoseDetectionModule) },
17+
{ path: 'mlkit-selfie-segmentation', loadChildren: () => import('./plugin-demos/mlkit-selfie-segmentation.module').then(m => m.MlkitSelfieSegmentationModule) },
1718
{ path: 'mlkit-text-recognition', loadChildren: () => import('./plugin-demos/mlkit-text-recognition.module').then(m => m.MlkitTextRecognitionModule) }
1819
];
1920

‎apps/demo-angular/src/home.component.ts

+3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ export class HomeComponent {
2727
{
2828
name: 'mlkit-pose-detection'
2929
},
30+
{
31+
name: 'mlkit-selfie-segmentation'
32+
},
3033
{
3134
name: 'mlkit-text-recognition'
3235
}

‎apps/demo-angular/src/plugin-demos/mlkit-selfie-segmentation.component.html

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<ActionBar title="mlkit-selfie-segmentation" class="action-bar"> </ActionBar>
2+
<StackLayout class="p-20">
3+
<ScrollView class="h-full">
4+
<StackLayout>
5+
<Button text="Test mlkit-selfie-segmentation" (tap)="demoShared.testIt()" class="btn btn-primary"></Button>
6+
</StackLayout>
7+
</ScrollView>
8+
</StackLayout>

‎apps/demo-angular/src/plugin-demos/mlkit-selfie-segmentation.component.ts

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import { Component, NgZone } from '@angular/core';
2+
import { DemoSharedMlkitSelfieSegmentation } from '@demo/shared';
3+
import { } from '@nativescript/mlkit-selfie-segmentation';
4+
5+
@Component({
6+
selector: 'demo-mlkit-selfie-segmentation',
7+
templateUrl: 'mlkit-selfie-segmentation.component.html',
8+
})
9+
export class MlkitSelfieSegmentationComponent {
10+
11+
demoShared: DemoSharedMlkitSelfieSegmentation;
12+
13+
constructor(private _ngZone: NgZone) {}
14+
15+
ngOnInit() {
16+
this.demoShared = new DemoSharedMlkitSelfieSegmentation();
17+
}
18+
19+
}

‎apps/demo-angular/src/plugin-demos/mlkit-selfie-segmentation.module.ts

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import { NgModule, NO_ERRORS_SCHEMA } from '@angular/core';
2+
import { NativeScriptCommonModule, NativeScriptRouterModule } from '@nativescript/angular';
3+
import { MlkitSelfieSegmentationComponent } from './mlkit-selfie-segmentation.component';
4+
5+
@NgModule({
6+
imports: [NativeScriptCommonModule, NativeScriptRouterModule.forChild([{ path: '', component: MlkitSelfieSegmentationComponent }])],
7+
declarations: [MlkitSelfieSegmentationComponent],
8+
schemas: [ NO_ERRORS_SCHEMA]
9+
})
10+
export class MlkitSelfieSegmentationModule {}

‎apps/demo/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,11 @@
1212
"@nativescript/mlkit-image-labeling": "file:../../packages/mlkit-image-labeling",
1313
"@nativescript/mlkit-object-detection": "file:../../packages/mlkit-object-detection",
1414
"@nativescript/mlkit-pose-detection": "file:../../packages/mlkit-pose-detection",
15+
"@nativescript/mlkit-selfie-segmentation": "file:../../packages/mlkit-selfie-segmentation",
1516
"@nativescript/mlkit-text-recognition": "file:../../packages/mlkit-text-recognition"
1617
},
1718
"devDependencies": {
18-
"@nativescript/android": "~8.1.1",
19+
"@nativescript/android": "8.1.1",
1920
"@nativescript/ios": "~8.1.0"
2021
}
2122
}

‎apps/demo/src/main-page.xml

+1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
<Button text="mlkit-object-detection" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>
1515
<Button text="mlkit-digital-ink-recognition" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>
1616
<Button text="mlkit-pose-detection" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>
17+
<Button text="mlkit-selfie-segmentation" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>
1718
</StackLayout>
1819
</ScrollView>
1920
</StackLayout>

‎apps/demo/src/plugin-demos/mlkit-core.ts

+16-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
import { Observable, EventData, Page, Dialogs } from '@nativescript/core';
1+
import { Observable, EventData, Page, Dialogs, ImageSource } from '@nativescript/core';
22
import { DemoSharedMlkitCore } from '@demo/shared';
3-
import { DetectionType, MLKitView, DetectionEvent } from '@nativescript/mlkit-core';
3+
import { DetectionType, MLKitView, DetectionEvent, detectWithStillImage } from '@nativescript/mlkit-core';
44
import { BarcodeResult } from '@nativescript/mlkit-barcode-scanning';
55
import { FaceResult } from '@nativescript/mlkit-face-detection';
66
import { ImageLabelingResult } from '@nativescript/mlkit-image-labeling';
@@ -58,5 +58,19 @@ export class DemoModel extends DemoSharedMlkitCore {
5858
this.camera.pause = !this.camera.pause;
5959
this.set('isPaused', this.camera.pause);
6060
}
61+
62+
async processStill(args) {
63+
try {
64+
const src = await ImageSource.fromUrl('https://www.jqueryscript.net/images/jQuery-Plugin-To-Generate-International-Article-Number-Barcode-EAN13.jpg');
65+
66+
console.log(src.android)
67+
const result = await detectWithStillImage(src, {
68+
detectorType: DetectionType.Barcode
69+
});
70+
console.log('processStill', result.barcode[0]);
71+
} catch (e) {
72+
console.log(e);
73+
}
74+
}
6175
}
6276

‎apps/demo/src/plugin-demos/mlkit-core.xml

+2-1
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@
33
<ActionBar title="mlkit-core" icon="" class="action-bar">
44
</ActionBar>
55
</Page.actionBar>
6-
<GridLayout rows="*,auto,auto,auto,auto,auto,auto" height="100%">
6+
<GridLayout rows="*,auto,auto,auto,auto,auto,auto,auto" height="100%">
77
<ui:MLKitView pause="true" height="100%" rowSpan="3" loaded="{{ onLoaded }}" cameraPosition="back" detectionType="{{ detectorType }}" detection="{{ onDetection }}"/>
88
<Button row="1" height="40" text="Toggle Camera" tap="{{ toggleCamera }}" />
99
<Button row="2" height="40" text="Request Camera Permission" tap="{{ requestPermission }}" />
1010
<Label row="3" text="{{'Detecting ' + detectorType }}"/>
1111
<Button row="4" height="40" text="Change Detector Type" tap="{{ changeType }}" />
1212
<Label row="5" text="{{'isPaused : ' + isPaused }}"/>
1313
<Button row="6" height="40" text="Toggle Pause" tap="{{ togglePause }}" />
14+
<Button row="7" height="40" text="Process Still imAGE" tap="{{ processStill }}" />
1415
</GridLayout>
1516
</Page>

‎apps/demo/src/plugin-demos/mlkit-selfie-segmentation.ts

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
import { Observable, EventData, Page } from '@nativescript/core';
2+
import { DemoSharedMlkitSelfieSegmentation } from '@demo/shared';
3+
import { } from '@nativescript/mlkit-selfie-segmentation';
4+
5+
export function navigatingTo(args: EventData) {
6+
const page = <Page>args.object;
7+
page.bindingContext = new DemoModel();
8+
}
9+
10+
export class DemoModel extends DemoSharedMlkitSelfieSegmentation {
11+
12+
}

‎apps/demo/src/plugin-demos/mlkit-selfie-segmentation.xml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<Page xmlns="http://schemas.nativescript.org/tns.xsd" navigatingTo="navigatingTo" class="page">
2+
<Page.actionBar>
3+
<ActionBar title="mlkit-selfie-segmentation" icon="" class="action-bar">
4+
</ActionBar>
5+
</Page.actionBar>
6+
<StackLayout class="p-20">
7+
<ScrollView class="h-full">
8+
<StackLayout>
9+
<Button text="Test mlkit-selfie-segmentation" tap="{{ testIt }}" class="btn btn-primary"/>
10+
11+
</StackLayout>
12+
</ScrollView>
13+
</StackLayout>
14+
</Page>

‎apps/demo/webpack.config.js

+5-5
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,11 @@ module.exports = (env) => {
1212
});
1313

1414
// Example if you need to share images across demo apps:
15-
// webpack.Utils.addCopyRule({
16-
// from: '../../../tools/images',
17-
// to: 'images',
18-
// context: webpack.Utils.project.getProjectFilePath('node_modules')
19-
// });
15+
webpack.Utils.addCopyRule({
16+
from: '../../../tools/images',
17+
to: 'images',
18+
context: webpack.Utils.project.getProjectFilePath('node_modules')
19+
});
2020

2121
return webpack.resolveConfig();
2222
};

‎nx.json

+3
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@
4646
},
4747
"mlkit-pose-detection": {
4848
"tags": []
49+
},
50+
"mlkit-selfie-segmentation": {
51+
"tags": []
4952
}
5053
},
5154
"workspaceLayout": {

‎packages/mlkit-barcode-scanning/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@nativescript/mlkit-barcode-scanning",
3-
"version": "1.0.0-alpha.1",
3+
"version": "1.0.0-alpha.2",
44
"description": "NativeScript MLKit Barcode Scanner module",
55
"main": "index",
66
"typings": "index.d.ts",

‎packages/mlkit-barcode-scanning/platforms/android/include.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
dependencies {
2-
implementation 'io.github.triniwiz.fancycamera:barcodeScanning:1.0.0-alpha0'
2+
implementation 'io.github.triniwiz.fancycamera:barcodeScanning:1.0.0-alpha.2'
33
}

‎packages/mlkit-core/common.ts

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ export enum DetectionType {
99
Pose = "pose",
1010
Text = "text",
1111
All = "all",
12+
Selfie = "selfie",
1213
None = "none"
1314
}
1415

‎packages/mlkit-core/index.android.ts

+96-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { BarcodeFormats, barcodeFormatsProperty, CameraPosition, cameraPositionProperty, DetectionType, detectionTypeProperty, faceDetectionMinFaceSizeProperty, faceDetectionPerformanceModeProperty, faceDetectionTrackingEnabledProperty, imageLabelerConfidenceThresholdProperty, MLKitViewBase, objectDetectionClassifyProperty, objectDetectionMultipleProperty, pauseProperty, torchOnProperty } from "./common";
2-
import { Application, Device, Utils, AndroidActivityRequestPermissionsEventData } from '@nativescript/core';
2+
import { Application, Device, Utils, AndroidActivityRequestPermissionsEventData, ImageSource } from '@nativescript/core';
33
import lazy from '@nativescript/core/utils/lazy';
4+
import { StillImageDetectionOptions } from ".";
45

56
const DetectorType_All = lazy(() => io.github.triniwiz.fancycamera.DetectorType.All);
67
const DetectorType_Barcode = lazy(() => io.github.triniwiz.fancycamera.DetectorType.Barcode);
@@ -11,6 +12,7 @@ const DetectorType_None = lazy(() => io.github.triniwiz.fancycamera.DetectorType
1112
const DetectorType_Object = lazy(() => io.github.triniwiz.fancycamera.DetectorType.Object);
1213
const DetectorType_Pose = lazy(() => io.github.triniwiz.fancycamera.DetectorType.Pose);
1314
const DetectorType_Text = lazy(() => io.github.triniwiz.fancycamera.DetectorType.Text);
15+
const DetectorType_Selfie = lazy(() => (io as any).github.triniwiz.fancycamera.DetectorType.Selfie);
1416

1517
const BARCODE_SCANNER_SUPPORTED = lazy(() => typeof io.github.triniwiz.fancycamera?.barcodescanning?.BarcodeScanner);
1618
const TEXT_RECOGNITION_SUPPORTED = lazy(() => typeof io.github.triniwiz.fancycamera?.textrecognition?.TextRecognition);
@@ -136,6 +138,9 @@ export class MLKitView extends MLKitViewBase {
136138
case DetectionType.Text:
137139
type = DetectorType_Text();
138140
break;
141+
case DetectionType.Selfie:
142+
type = DetectorType_Selfie();
143+
break;
139144
default:
140145
type = DetectorType_None();
141146
break;
@@ -490,4 +495,93 @@ export class MLKitView extends MLKitViewBase {
490495
hasCameraPermission(): boolean {
491496
return this.#camera.hasCameraPermission();
492497
}
493-
}
498+
}
499+
500+
export function detectWithStillImage(image: any, options?: StillImageDetectionOptions) {
501+
return new Promise((resolve, reject) => {
502+
let nativeImage;
503+
let rotation = 0;
504+
if (image instanceof ImageSource) {
505+
nativeImage = image.android;
506+
rotation = image.rotationAngle;
507+
} else if (image instanceof android.graphics.Bitmap) {
508+
nativeImage = image;
509+
} else {
510+
reject('Please use a valid Image');
511+
}
512+
513+
514+
515+
let type = 9 /* None */
516+
switch (options?.detectorType) {
517+
case DetectionType.All:
518+
type = 7;
519+
break;
520+
case DetectionType.Barcode:
521+
type = 0;
522+
break;
523+
case DetectionType.DigitalInk:
524+
type = 1
525+
break;
526+
case DetectionType.Face:
527+
type = 2
528+
break;
529+
case DetectionType.Image:
530+
type = 3
531+
break;
532+
case DetectionType.Object:
533+
type = 4
534+
break;
535+
case DetectionType.Pose:
536+
type = 5
537+
break;
538+
case DetectionType.Text:
539+
type = 6
540+
break;
541+
case DetectionType.Selfie:
542+
type = 8;
543+
break;
544+
default:
545+
type = 9;
546+
break;
547+
}
548+
549+
550+
(io as any).github.triniwiz.fancycamera.ML.processImage(nativeImage, rotation || 0, JSON.stringify({
551+
...{
552+
barcodeScanning: {
553+
barcodeFormat: [0]
554+
},
555+
faceDetection: {},
556+
imageLabeling: {},
557+
objectDetection: {},
558+
selfieSegmentation: {}
559+
}, ...options, ...{ detectorType: type }
560+
}), new io.github.triniwiz.fancycamera.ImageAnalysisCallback({
561+
onSuccess(param0: any) {
562+
const results = {}
563+
const size = param0.size();
564+
for (let i = 0; i < size; i++) {
565+
const item = param0.get(i);
566+
const type = item[0];
567+
const result = item[1];
568+
try {
569+
if (type.toString() === DetectionType.Selfie) {
570+
results[type] = {
571+
width: result.geWidth(),
572+
height: result.getHeight(),
573+
buffer: (ArrayBuffer as any).from(result.getBuffer())
574+
}
575+
} else {
576+
results[type] = JSON.parse(result.toString());
577+
}
578+
} catch (e) { }
579+
}
580+
resolve(results);
581+
},
582+
onError(param0: string, param1: java.lang.Exception) {
583+
reject(param0)
584+
}
585+
}))
586+
})
587+
}

‎packages/mlkit-core/index.d.ts

+31-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,33 @@
1-
import { MLKitViewBase, DetectionType } from "./common";
1+
import { MLKitViewBase, DetectionType, BarcodeFormats } from "./common";
22
import { EventData } from '@nativescript/core';
3+
4+
5+
export interface StillImageDetectionOptions {
6+
detectorType: DetectionType
7+
barcodeScanning?: {
8+
barcodeFormat?: [BarcodeFormats]
9+
}
10+
faceDetection?: {
11+
faceTracking?: boolean
12+
minimumFaceSize: ?number
13+
detectionMode?: "fast" | "accurate"
14+
landmarkMode?: "all" | "none"
15+
contourMode?: "all" | "none"
16+
classificationMode?: "all" | "none"
17+
}
18+
imageLabeling?: {
19+
confidenceThreshold?: number
20+
}
21+
objectDetection?: {
22+
multiple: boolean;
23+
classification: boolean;
24+
}
25+
selfieSegmentation?: {
26+
enableRawSizeMask?: boolean;
27+
smoothingRatio?: number;
28+
}
29+
}
30+
331
export interface DetectionEvent extends EventData {
432
data: { [key: string]: any };
533
type: DetectionType
@@ -15,3 +43,5 @@ export declare class MLKitView extends MLKitViewBase {
1543
hasCameraPermission(): boolean;
1644
on(event: 'detection', callback: (args: DetectionEvent) => void, thisArg?: any);
1745
}
46+
47+
export function detectWithStillImage(image: any, options?: StillImageDetectionOptions): Promise<{ [key: string]: any }>

0 commit comments

Comments
 (0)