Skip to content

Commit 7829ece

Browse files
JingyuanZhangJingyuanZhang
authored
Merge pull request #227 from yueshuangyan/master
feat: adapt wx environment
2 parents afe7e12 + 54055c1 commit 7829ece

File tree

18 files changed

+211
-85
lines changed

18 files changed

+211
-85
lines changed

‎packages/paddlejs-backend-webgl/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@paddlejs/paddlejs-backend-webgl",
3-
"version": "1.0.3",
3+
"version": "1.0.4",
44
"description": "",
55
"main": "lib/index",
66
"scripts": {

‎packages/paddlejs-backend-webgl/src/backend.ts‎

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,14 @@ export default class WebGLBackend extends PaddlejsBackend {
5959
async init() {
6060
// 初始化webgl环境
6161
const gl = this.gl = GLHelper.createWebGLRenderingContext();
62+
if (!this.gl) {
63+
return;
64+
}
6265
this.glVersion = GLHelper.getWebglVersion();
66+
67+
// texture conf
68+
this.textureConf = GLTexture.getTextureConfig(gl);
69+
this.MAX_TEXTURE_SIZE = gl.getParameter(gl.MAX_TEXTURE_SIZE);
6370
// 关闭相关功能
6471
gl.disable(gl.DEPTH_TEST);
6572
gl.disable(gl.STENCIL_TEST);
@@ -81,9 +88,6 @@ export default class WebGLBackend extends PaddlejsBackend {
8188
gl.bindFramebuffer(gl.FRAMEBUFFER, this.frameBuffer);
8289
// pbo
8390
this.pbo = gl.createBuffer();
84-
// texture conf
85-
this.textureConf = GLTexture.getTextureConfig(gl);
86-
this.MAX_TEXTURE_SIZE = this.gl.getParameter(this.gl.MAX_TEXTURE_SIZE);
8791
}
8892

8993

@@ -324,19 +328,20 @@ export default class WebGLBackend extends PaddlejsBackend {
324328
isPacked: Boolean = false
325329
) {
326330
const {
327-
inputTensors: data,
331+
inputTensors: data = [],
328332
uniform = null,
329333
iLayer = 0,
330334
modelName
331335
} = opData;
332336
const gl = this.gl;
333337
let textureIndex = 0;
338+
334339
data.forEach(item => {
340+
this.initTexture(textureIndex, item, isPacked);
335341
const loc = this.getUniformLoc('texture_' + item.name, iLayer, isRendered, index, modelName);
336342
if (!loc) {
337343
return;
338344
}
339-
this.initTexture(textureIndex, item, isPacked);
340345
gl.uniform1i(loc, textureIndex++);
341346
});
342347
if (uniform) {
@@ -369,14 +374,42 @@ export default class WebGLBackend extends PaddlejsBackend {
369374
}
370375
gl.activeTexture(gl[`TEXTURE${index}`]);
371376
gl.bindTexture(gl.TEXTURE_2D, texture);
377+
const data = item.data;
372378

373-
if (item.data) {
379+
if (data) {
374380
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
375381
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
376382
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
377383
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
378384

379-
if (this.glVersion === 2) {
385+
if (data instanceof Uint8Array || data instanceof Uint8ClampedArray) {
386+
// 输入为0~255之间的像素数据，类型为Uint8Array 或 Uint8ClampedArray
387+
gl.texImage2D(
388+
gl.TEXTURE_2D,
389+
0,
390+
gl.RGBA,
391+
item.width_texture,
392+
item.height_texture,
393+
0,
394+
gl.RGBA,
395+
gl.UNSIGNED_BYTE,
396+
data
397+
);
398+
}
399+
else if (!(data instanceof Float32Array || data instanceof Array)) {
400+
// 输入数据不是数组类型，包括Array、Float32Array、Uint8Array和Uint8ClampedArray
401+
// 输入数据可能是HTMLImageElement、HTMLVideoElement、HTMLCanvasElement、小程序中图像的临时path string。
402+
gl.texImage2D(
403+
gl.TEXTURE_2D,
404+
0,
405+
gl.RGBA,
406+
gl.RGBA,
407+
gl.UNSIGNED_BYTE,
408+
data
409+
);
410+
}
411+
else if (this.glVersion === 2) {
412+
// 输入数据类型是Float32Array
380413
const useHalfFloat = env.get('webgl_force_half_float_texture');
381414
const internalFormat = packed
382415
? useHalfFloat
@@ -397,10 +430,11 @@ export default class WebGLBackend extends PaddlejsBackend {
397430
0,
398431
textureFormat,
399432
gl.FLOAT,
400-
item.data
433+
data as Float32Array
401434
);
402435
}
403436
else {
437+
// 输入数据类型是Float32Array
404438
const temp = new Float32Array(item.width_texture * item.height_texture * 4);
405439
for (let i = 0; i < item.data.length; i++) {
406440
if (packed) {

‎packages/paddlejs-backend-webgl/src/ops/shader/custom/feedPost.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ function mainFunc(
1313
float res = 0.0;
1414
int c1 = int(mod(float(oPos[1]), 4.0));
1515
int c = oPos[1];
16-
vec4 o = getValueFromTensorPosPacking_origin(oPos[0], c / 4, oPos[2], oPos[3]) / 255.0;
16+
vec4 o = getValueFromTensorPosPacking_origin(oPos[0], c / 4, oPos[2], oPos[3]);
1717
1818
if (c1 == 0) {
1919
res = o.r;

‎packages/paddlejs-backend-webgl/src/ops/shader/custom/imgFeed.ts‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,21 @@
33
*/
44

55
function mainFunc() {
6-
76
return `
8-
7+
uniform vec2 u_scale;
98
void main(void) {
109
vec2 outCoord = vCoord.xy;
11-
vec4 origin = TEXTURE2D(texture_origin, vCoord.xy);
12-
setPackedOutput(origin);
10+
vec2 newPos = vCoord / u_scale;
11+
vec2 startPos = (1.0 / u_scale - 1.0) / 2.0;
12+
bool exceedX = u_scale.y == 1.0 && (newPos.x < startPos.x || newPos.x > (1.0 + startPos.x));
13+
bool exceedY = u_scale.x == 1.0 && (newPos.y < startPos.y || newPos.y > (1.0 + startPos.y));
14+
if (exceedX || exceedY) {
15+
setPackedOutput(vec4(1.0, 1.0, 1.0, 1.0));
16+
return;
17+
}
18+
newPos = newPos - startPos;
19+
vec4 counter = TEXTURE2D(texture_origin, newPos);
20+
setPackedOutput(counter);
1321
}
1422
`;
1523
}

‎packages/paddlejs-backend-webgl/src/webgl/WebGLTexture.ts‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,13 @@ export class GLTexture {
5353
const TEXTURE_FLOAT = 'OES_texture_float';
5454
const TEXTURE_HALF_FLOAT = 'OES_texture_half_float';
5555
textureFloat = gl.getExtension(TEXTURE_FLOAT);
56-
textureHalfFloat = gl.getExtension(TEXTURE_HALF_FLOAT);
56+
textureHalfFloat = gl.getExtension(TEXTURE_HALF_FLOAT).HALF_FLOAT_OES;
5757
frameBufferSupportFloat = this.isDownloadFloatTextureEnabled(gl, downloadInternalFormat);
58-
isFloatTextureReadPixelsEnabled = this.isFloatTextureReadPixelsEnabledMethod(gl, 1);
58+
isFloatTextureReadPixelsEnabled = this.isFloatTextureReadPixelsEnabledMethod(
59+
gl,
60+
1, // gl version
61+
frameBufferSupportFloat
62+
);
5963
}
6064

6165
return {
@@ -72,7 +76,11 @@ export class GLTexture {
7276
};
7377
}
7478

75-
public static isFloatTextureReadPixelsEnabledMethod(webgl: WebGLRenderingContext, webGLVersion) {
79+
public static isFloatTextureReadPixelsEnabledMethod(
80+
webgl: WebGLRenderingContext,
81+
webGLVersion,
82+
frameBufferSupportFloat
83+
) {
7684
const gl = webgl as any;
7785
if (webGLVersion === 0) {
7886
return false;
@@ -104,7 +112,7 @@ export class GLTexture {
104112
1,
105113
0,
106114
gl.RGBA,
107-
gl.getExtension('OES_texture_half_float').HALF_FLOAT_OES,
115+
frameBufferSupportFloat ? gl.FLOAT : gl.getExtension('OES_texture_half_float').HALF_FLOAT_OES,
108116
null
109117
);
110118

@@ -118,7 +126,6 @@ export class GLTexture {
118126

119127
const readPixelsNoError = gl.getError() === gl.NO_ERROR;
120128

121-
122129
return frameBufferComplete && readPixelsNoError;
123130
}
124131

@@ -228,7 +235,7 @@ export class GLTexture {
228235
: gl.FLOAT
229236
: textureConf.frameBufferSupportFloat
230237
? gl.FLOAT
231-
: textureConf.textureHalfFloat.HALF_FLOAT_OES;
238+
: textureConf.textureHalfFloat;
232239

233240
const textureTypeForReadPixel = isFinalOp
234241
? textureConf.isFloatTextureReadPixelsEnabled

‎packages/paddlejs-backend-webgl/src/webgl/WebGLUtils.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ export class GLHelper {
8484
}
8585

8686
public static createCanvas() {
87-
return document.createElement('canvas');
87+
return env.get('canvas') || (document && document.createElement('canvas'));
8888
}
8989

9090
public static setWebglVersion(version: number) {
@@ -110,7 +110,11 @@ export class GLHelper {
110110

111111
const canvas = this.createCanvas();
112112

113-
canvas.addEventListener('webglcontextlost', (ev: Event) => {
113+
if (!canvas) {
114+
return null;
115+
}
116+
117+
canvas.addEventListener && canvas.addEventListener('webglcontextlost', (ev: Event) => {
114118
ev.preventDefault();
115119
throw Error('webgl context is lost~');
116120
}, false);

‎packages/paddlejs-core/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@paddlejs/paddlejs-core",
3-
"version": "2.0.3",
3+
"version": "2.0.4",
44
"description": "",
55
"main": "lib/index",
66
"scripts": {

‎packages/paddlejs-core/src/commons/interface.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ export interface ModelOp {
1414
isPacked?: boolean;
1515
bufferType?: BufferType;
1616
uniform?: OpUniform | null;
17+
scale?: number[];
18+
pos?: number[];
1719
}
1820

1921
export interface ModelVar {
@@ -169,7 +171,7 @@ export interface InputFeed {
169171
data: Float32Array | number[];
170172
shape: number[];
171173
name: string;
172-
persistable: boolean;
174+
persistable?: boolean;
173175
}
174176

175177

‎packages/paddlejs-core/src/loader.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,10 @@ export default class ModelLoader {
7777
return modelInfo;
7878
}
7979

80-
fetchOneChunk(path: string) {
80+
async fetchOneChunk(path: string) {
81+
if (env.get('fetch')) {
82+
return await env.get('fetch')(path, { type: 'arrayBuffer' });
83+
};
8184
return this.fetch(path).then(request => {
8285
return request.arrayBuffer();
8386
});
@@ -94,7 +97,7 @@ export default class ModelLoader {
9497
return `chunk_${i}.dat`;
9598
}
9699

97-
fetchChunks() {
100+
async fetchChunks() {
98101
const counts = this.chunkNum;
99102
const chunkArray: any[] = [];
100103
for (let i = 1; i <= counts; i++) {
@@ -139,6 +142,9 @@ export default class ModelLoader {
139142
}
140143

141144
fetch(path: string, params?: FetchParams) {
145+
if (env.get('fetch')) {
146+
return env.get('fetch')(path, params || {});
147+
}
142148
const fetchParams = params || this.params;
143149
const method = fetchParams.method || 'get';
144150
const HeadersClass = this.inNode
@@ -179,6 +185,9 @@ export default class ModelLoader {
179185
load = new Promise((resolve, reject) => {
180186
this.fetch(path, params)
181187
.then(response => {
188+
if (env.get('fetch')) {
189+
return response;
190+
}
182191
return this.isLocalFile && this.inNode
183192
? JSON.parse(response)
184193
: response.json();

‎packages/paddlejs-core/src/mediaProcessor.ts‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ type Color = string;
99

1010
export default class MediaProcessor {
1111
targetContext: CanvasRenderingContext2D = {} as CanvasRenderingContext2D;
12+
targetCanvas: HTMLCanvasElement;
1213
gapFillWith: Color = '#fff';
1314
mean: number[] = [0, 0, 0];
1415
std: number[] = [1, 1, 1];
@@ -18,8 +19,8 @@ export default class MediaProcessor {
1819
inputFeed: InputFeed[] = [];
1920

2021
constructor() {
21-
const targetCanvas = document.createElement('canvas') as HTMLCanvasElement;
22-
this.targetContext = targetCanvas.getContext('2d') as CanvasRenderingContext2D;
22+
this.targetCanvas = env.get('canvas2d') || document.createElement('canvas') as HTMLCanvasElement;
23+
this.targetContext = this.targetCanvas.getContext('2d') as CanvasRenderingContext2D;
2324
};
2425

2526
/**
@@ -57,7 +58,9 @@ export default class MediaProcessor {
5758
dHeight: opt.targetSize.height
5859
};
5960

60-
if (!(pixels instanceof HTMLImageElement
61+
const input = pixels;
62+
const isImageElementLike = pixels.path && pixels.width && pixels.height;
63+
if (!isImageElementLike && !(pixels instanceof HTMLImageElement
6164
|| pixels instanceof HTMLVideoElement
6265
|| pixels instanceof HTMLCanvasElement)) {
6366
return [{
@@ -72,7 +75,7 @@ export default class MediaProcessor {
7275
this.pixelHeight = (pixels as HTMLImageElement).naturalHeight || pixels.height;
7376

7477
const inGPU = env.get('webgl_gpu_pipeline') || env.get('webgl_feed_process');
75-
this.fitToTargetSize(pixels, imageDataInfo, inGPU);
78+
this.fitToTargetSize(isImageElementLike ? input.path : input, imageDataInfo, inGPU);
7679
data = this.getImageData(imageDataInfo);
7780
// process imageData in webgl
7881
if (inGPU) {
@@ -171,8 +174,8 @@ export default class MediaProcessor {
171174

172175
imageDataInfo.dWidth = canvasWidth;
173176
imageDataInfo.dHeight = canvasHeight;
174-
this.targetContext.canvas.width = canvasWidth;
175-
this.targetContext.canvas.height = canvasHeight;
177+
this.targetCanvas.width = canvasWidth;
178+
this.targetCanvas.height = canvasHeight;
176179
this.targetContext.fillStyle = imageDataInfo.gapFillWith;
177180
this.targetContext.fillRect(0, 0, canvasHeight, canvasWidth);
178181
this.targetContext.drawImage(image, x, y, sw, sh);
@@ -190,4 +193,20 @@ export default class MediaProcessor {
190193
return this.targetContext.getImageData(dx, dy, dWidth, dHeight);
191194
}
192195

196+
cover(w, h, dw, dh) {
197+
// 缩放后的宽高
198+
let sw = dw;
199+
let sh = dh;
200+
// target的长宽比大些 就把原图的高变成target那么高
201+
if (dw / dh * h / w >= 1) {
202+
sw = Math.round(sh * w / h);
203+
}
204+
// target的长宽比小些 就把原图的宽变成target那么宽
205+
else {
206+
sh = Math.round(sw * h / w);
207+
}
208+
209+
const scale = [sw / dw, sh / dh];
210+
return scale;
211+
}
193212
}

0 commit comments

Comments
 (0)