2 #include "../core/GLContext.h" 3 #include "../core/GLFilter.h" 4 #include "../core/GLImage.h" 5 #include "../core/ImageAllocator.h" 7 #include "caffe2/core/common.h" 8 #include "caffe2/core/context.h" 9 #include "caffe2/core/operator.h" 11 enum InputFormat { BGRA = 0, RGBA = 1 };
21 GLStylizer(
bool _deprocess =
false, InputFormat input_format = BGRA)
22 :
GLFilter(_deprocess ?
"GLDeStylizer" :
"GLStylizer",
25 std::vector<binding*>({BINDING(inputData), BINDING(mean), BINDING(noise_std), BINDING(outputSize)}),
28 {{
"DEPROCESS", caffe2::to_string(_deprocess)}, {
"RGBAINPUT", caffe2::to_string(input_format)}}),
29 deprocess(_deprocess) {}
31 template <
typename T1,
typename T2>
34 const float mean_values[3],
35 float noise_std_value);
37 static const char* fragment_shader;
42 const char* GLStylizer::fragment_shader = R
"GLSL(#version 300 es 44 #define DEPROCESS $(DEPROCESS) 45 #define RGBAINPUT $(RGBAINPUT) 47 precision mediump float; 48 precision mediump int; 49 precision mediump sampler2D; 51 in highp vec2 v_texCoord; 53 uniform ivec2 outputSize; 56 uniform float noise_std; 59 TEXTURE_INPUT(inputData); 60 layout(location = 0) out mediump vec4 outputData; 62 uniform sampler2D inputData; 63 TEXTURE_OUTPUT(0, outputData); 67 // http://byteblacksmith.com/improvements-to-the-canonical-one-liner-glsl-rand-for-opengl-es-2-0/ 69 highp float rand(vec2 co) { 70 highp float a = 12.9898; 71 highp float b = 78.233; 72 highp float c = 43758.5453; 73 highp float dt = dot(co.xy, vec2(a, b)); 74 highp float sn = mod(dt, 3.14); 75 return fract(sin(sn) * c); 79 // In AR Engine, input/output a RBGA texture; otherwise, BGRA tensor => texture 83 ivec2 texelCoord = ivec2(v_texCoord * vec2(outputSize)); 84 vec4 val = TEXTURE_LOAD(inputData, texelCoord); 85 outputData = vec4((val.rgb + mean) / 255.0, 1.0).bgra; 87 outputData = TEXTURE_STORE(vec4(255.0 * texture(inputData, v_texCoord).bgr - mean + vec3(noise_std * rand(v_texCoord)), 0.0)); 93 ivec2 texelCoord = ivec2(v_texCoord * vec2(outputSize)); 94 vec4 val = TEXTURE_LOAD(inputData, texelCoord); 95 outputData = vec4((val.rgb + mean) / 255.0, 1.0); 97 outputData = TEXTURE_STORE(vec4(255.0 * texture(inputData, v_texCoord).rgb - mean + vec3(noise_std * rand(v_texCoord)), 0.0)); 103 template <
typename T1,
typename T2>
104 void GLStylizer::stylize(
const GLImage<T1>* input_image,
106 const float mean_values[3],
107 float noise_std_value) {
108 int input_slices = input_image->slices;
109 int output_slices = output_image->slices;
111 run(std::vector<texture_attachment>({{input_image->textures[0], inputData}}),
112 {output_image->textures[0]},
114 glUniform2i(outputSize->location, output_image->width, output_image->height);
115 glUniform3f(mean->location, mean_values[0], mean_values[1], mean_values[2]);
117 glUniform1f(noise_std->location, noise_std_value);
121 output_image->height);
130 static constexpr
int kInputChannels = 4;
133 static constexpr
int kOutputChannels = 3;
135 USE_OPERATOR_BASE_FUNCTIONS;
141 const auto& input = Input(0);
142 const auto& mean = Input(1);
144 CAFFE_ENFORCE(input.ndim() == 4);
146 const int num_images = input.dim32(0);
147 const int input_height = input.dim32(1);
148 const int input_width = input.dim32(2);
149 const int input_channels = input.dim32(3);
151 CAFFE_ENFORCE(input.dim32(0) == 1);
152 CAFFE_ENFORCE(input_channels == kInputChannels);
153 CAFFE_ENFORCE(mean.size() == kOutputChannels);
156 const float* mean_buffer = mean.template data<float>();
157 const uint8_t* input_buffer = input.template data<uint8_t>();
160 GLContext::getGLContext()->set_context();
172 const int tile_x = 1, tile_y = 1;
174 num_images, input_width, input_height, kInputChannels, tile_x, tile_y,
false);
175 for (
int i = 0; i < num_images; i++) {
176 auto input_image = (*input_images)[i];
177 auto output_image = (*output_images)[i];
178 const GLTexture* inputTexture = input_image->textures[0];
179 inputTexture->loadData(input_buffer);
185 glStylizer_->stylize(
186 input_image, output_image, mean_buffer, GetSingleArgument<float>(
"noise_std", 10.0));
189 Outputs()[0]->Reset(output_images);
195 std::unique_ptr<GLStylizer> glStylizer_;
198 template <InputFormat inputFormat>
204 static constexpr
int kInputChannels = 4;
207 static constexpr
int kOutputChannels = 3;
209 USE_OPERATOR_BASE_FUNCTIONS;
216 const auto& mean = Input(1);
218 const int num_images = input.size();
219 const int input_height = input.height();
220 const int input_width = input.width();
221 const int input_channels = input.channels();
223 CAFFE_ENFORCE_GT(num_images, 0);
224 CAFFE_ENFORCE(input[0]->slices == 1);
225 CAFFE_ENFORCE(input_channels == kInputChannels);
226 CAFFE_ENFORCE(mean.size() == kOutputChannels);
229 const float* mean_buffer = mean.template data<float>();
232 num_images, input_width, input_height, kOutputChannels,
false);
235 glStylizer_.reset(
new GLStylizer(
false, inputFormat));
237 for (
int i = 0; i < num_images; i++) {
238 auto input_image = input[i];
239 auto output_image = (*output_images)[i];
240 glStylizer_->stylize(
241 input_image, output_image, mean_buffer, GetSingleArgument<float>(
"noise_std", 10.0));
243 Outputs()[0]->Reset(output_images);
249 std::unique_ptr<GLStylizer> glStylizer_;
252 REGISTER_CPU_OPERATOR(OpenGLTensorToTextureStylizerPreprocess,
254 OPERATOR_SCHEMA(OpenGLTensorToTextureStylizerPreprocess).NumInputs(2).NumOutputs(1);
256 REGISTER_CPU_OPERATOR(OpenGLTextureToTextureStylizerPreprocess,
258 OPERATOR_SCHEMA(OpenGLTextureToTextureStylizerPreprocess).NumInputs(2).NumOutputs(1);
266 static constexpr
int kInputChannels = 3;
269 static constexpr
int kOutputChannels = 4;
273 const auto& mean = Input(1);
274 auto* output = Output(0);
276 const int num_images = input.size(), channels = input.channels(), height = input.height(),
277 width = input.width();
279 CAFFE_ENFORCE(mean.size() == kInputChannels);
280 CAFFE_ENFORCE(channels == kInputChannels);
282 output->Resize(num_images, height, width, kOutputChannels);
284 const auto* mean_data = mean.template data<float>();
285 auto* output_buffer = output->template mutable_data<uint8_t>();
294 for (
int i = 0; i < num_images; i++) {
295 auto input_image = input[i];
296 auto output_image = (*output_images)[i];
297 glStylizer_->stylize(input_image, output_image, mean_data, 0);
299 output_image->textures[0]->map_read([&](
const void* buffer,
305 if (width == stride) {
306 memcpy(output_buffer, buffer, channels * width * height);
308 typedef uint8_t(input_data_t)[height][stride][channels];
309 typedef uint8_t(output_data_t)[height][width][channels];
311 const input_data_t& input_data = *
reinterpret_cast<const input_data_t*
>(buffer);
312 output_data_t& output_data = *
reinterpret_cast<output_data_t*
>(output_buffer);
314 for (
int y = 0; y < height; y++) {
315 memcpy(output_data[y], input_data[y], channels * width);
320 delete output_images;
326 std::unique_ptr<GLStylizer> glStylizer_;
329 template <InputFormat inputFormat>
336 static constexpr
int kInputChannels = 3;
339 static constexpr
int kOutputChannels = 4;
343 const auto& mean = Input(1);
345 const int num_images = input.size(), channels = input.channels(), height = input.height(),
346 width = input.width();
348 CAFFE_ENFORCE(mean.size() == kInputChannels);
349 CAFFE_ENFORCE(channels == kInputChannels);
351 const auto* mean_data = mean.template data<float>();
356 auto textureAllocator = GLContext::getGLContext()->getTextureAllocator();
357 const int tile_x = 1, tile_y = 1;
358 if (textureAllocator !=
nullptr) {
360 num_images, width, height, kOutputChannels, tile_x, tile_y, textureAllocator);
367 glStylizer_.reset(
new GLStylizer(
true, inputFormat));
370 for (
int i = 0; i < num_images; i++) {
371 auto input_image = input[i];
372 auto output_image = (*output_images)[i];
373 glStylizer_->stylize(input_image, output_image, mean_data, 0);
376 Outputs()[0]->Reset(output_images);
382 std::unique_ptr<GLStylizer> glStylizer_;
385 REGISTER_CPU_OPERATOR(OpenGLTextureToTensorStylizerDeprocess,
387 OPERATOR_SCHEMA(OpenGLTextureToTensorStylizerDeprocess).NumInputs(2).NumOutputs(1);
389 REGISTER_CPU_OPERATOR(OpenGLTextureToTextureStylizerDeprocess,
391 OPERATOR_SCHEMA(OpenGLTextureToTextureStylizerDeprocess).NumInputs(2).NumOutputs(1);
Workspace is a class that holds all the related objects created during runtime: (1) all blobs...
A global dictionary that holds information about what Caffe2 modules have been loaded in the current ...