Browse Source

Update i420 photo

camera_next
Vitaliy Zarubin 1 year ago
parent
commit
83a1338b61
  1. 5
      example/lib/packages/camera/page.dart
  2. 25
      packages/camera/camera_aurora/aurora/camera_aurora_plugin.cpp
  3. 1
      packages/camera/camera_aurora/aurora/include/camera_aurora/camera_aurora_plugin.h
  4. 47
      packages/camera/camera_aurora/aurora/include/camera_aurora/texture_camera.h
  5. 330
      packages/camera/camera_aurora/aurora/texture_camera.cpp
  6. 6
      packages/camera/camera_aurora/lib/camera_aurora.dart
  7. 15
      packages/camera/camera_aurora/lib/camera_aurora_method_channel.dart
  8. 4
      packages/camera/camera_aurora/lib/camera_aurora_platform_interface.dart
  9. 6
      packages/camera/camera_aurora/lib/camera_data.dart
  10. 61
      packages/camera/camera_aurora/lib/camera_viewfinder.dart

5
example/lib/packages/camera/page.dart

@ -157,6 +157,11 @@ class _CameraPageState extends AppState<CameraPage> {
content: Text("File save to: ${file.path}"),
backgroundColor: AppColors.secondary,
));
} else {
ScaffoldMessenger.of(context).showSnackBar(SnackBar(
content: const Text("Error save file"),
backgroundColor: Colors.red[300],
));
}
}
}

25
packages/camera/camera_aurora/aurora/camera_aurora_plugin.cpp

@ -23,6 +23,7 @@ namespace CameraAuroraMethods
constexpr auto AvailableCameras = "availableCameras";
constexpr auto CreateCamera = "createCamera";
constexpr auto ResizeFrame = "resizeFrame";
constexpr auto Dispose = "dispose";
constexpr auto StartCapture = "startCapture";
constexpr auto StopCapture = "stopCapture";
@ -75,6 +76,11 @@ void CameraAuroraPlugin::RegisterMethods(PluginRegistrar &registrar)
onCreateCamera(call);
return;
}
if (method == CameraAuroraMethods::ResizeFrame)
{
onResizeFrame(call);
return;
}
if (method == CameraAuroraMethods::Dispose)
{
onDispose(call);
@ -156,6 +162,18 @@ void CameraAuroraPlugin::onCreateCamera(const MethodCall &call)
call.SendSuccessResponse(state);
}
void CameraAuroraPlugin::onResizeFrame(const MethodCall &call)
{
auto width = call.GetArgument<Encodable::Int>("width");
auto height = call.GetArgument<Encodable::Int>("height");
auto state = m_textureCamera->ResizeFrame(width, height);
EventChannel(CameraAuroraEvents::StateChanged, MethodCodecType::Standard).SendEvent(state);
unimplemented(call);
}
void CameraAuroraPlugin::onDispose(const MethodCall &call)
{
auto state = m_textureCamera->Unregister();
@ -167,7 +185,10 @@ void CameraAuroraPlugin::onDispose(const MethodCall &call)
void CameraAuroraPlugin::onStartCapture(const MethodCall &call)
{
auto state = m_textureCamera->StartCapture();
auto width = call.GetArgument<Encodable::Int>("width");
auto height = call.GetArgument<Encodable::Int>("height");
auto state = m_textureCamera->StartCapture(width, height);
EventChannel(CameraAuroraEvents::StateChanged, MethodCodecType::Standard).SendEvent(state);
@ -183,7 +204,7 @@ void CameraAuroraPlugin::onStopCapture(const MethodCall &call)
void CameraAuroraPlugin::onTakePicture(const MethodCall &call)
{
unimplemented(call);
call.SendSuccessResponse(m_textureCamera->GetImageBase64());
}
void CameraAuroraPlugin::onStartVideoRecording(const MethodCall &call)

1
packages/camera/camera_aurora/aurora/include/camera_aurora/camera_aurora_plugin.h

@ -29,6 +29,7 @@ private:
void onAvailableCameras(const MethodCall &call);
void onCreateCamera(const MethodCall &call);
void onResizeFrame(const MethodCall &call);
void onDispose(const MethodCall &call);
void onStartCapture(const MethodCall &call);
void onStopCapture(const MethodCall &call);

47
packages/camera/camera_aurora/aurora/include/camera_aurora/texture_camera.h

@ -8,8 +8,24 @@
#include <flutter/plugin-interface.h>
#include <streamcamera/streamcamera.h>
#include <QImage>
#include <QtCore>
typedef std::function<void()> CameraErrorHandler;
struct ResultYUV
{
uint8_t *y;
int strideY;
uint8_t *u;
int strideU;
uint8_t *v;
int strideV;
int width;
int height;
std::shared_ptr<uint8_t> raw;
};
class TextureCamera : public Aurora::StreamCamera::CameraListener
{
public:
@ -23,21 +39,33 @@ public:
std::vector<Encodable> GetAvailableCameras();
std::map<Encodable, Encodable> Register(std::string cameraName);
std::map<Encodable, Encodable> Unregister();
std::map<Encodable, Encodable> StartCapture();
std::map<Encodable, Encodable> StartCapture(size_t width, size_t height);
void StopCapture();
std::map<Encodable, Encodable> GetState();
std::string GetImageBase64();
std::map<Encodable, Encodable> ResizeFrame(size_t width, size_t height);
private:
bool CreateCamera(std::string cameraName);
void SendError(std::string error);
void ScaleToReqYuvCrop(
std::shared_ptr<const Aurora::StreamCamera::YCbCrFrame> frame,
std::shared_ptr<uint8_t> yPtr,
std::shared_ptr<uint8_t> uPtr,
std::shared_ptr<uint8_t> vPtr,
int ww,
int hh,
int stride
void ResizeFrame(size_t width, size_t height, Aurora::StreamCamera::CameraInfo info, Aurora::StreamCamera::CameraCapability cap);
std::optional<std::shared_ptr<const Aurora::StreamCamera::YCbCrFrame>> GetFrame(std::shared_ptr<Aurora::StreamCamera::GraphicBuffer> buffer);
ResultYUV YUVI420Scale(
const uint8_t* srcY,
const uint8_t* srcU,
const uint8_t* srcV,
int srcWidth,
int srcHeight,
int outWidth,
int outHeight
);
ResultYUV YUVI420Rotate(
const uint8_t* srcY,
const uint8_t* srcU,
const uint8_t* srcV,
int srcWidth,
int srcHeight,
int degree // 0, 90, 180, 270
);
private:
@ -48,6 +76,7 @@ private:
Aurora::StreamCamera::CameraManager *m_manager;
std::shared_ptr<Aurora::StreamCamera::Camera> m_camera;
std::shared_ptr<const Aurora::StreamCamera::YCbCrFrame> m_frame;
int64_t m_textureId = 0;
size_t m_captureWidth = 0;

330
packages/camera/camera_aurora/aurora/texture_camera.cpp

@ -22,8 +22,49 @@ TextureCamera::TextureCamera(TextureRegistrar *plugin, const CameraErrorHandler
: m_plugin(plugin)
, m_onError(onError)
, m_manager(StreamCameraManager())
, m_camera(nullptr)
{}
std::string TextureCamera::GetImageBase64()
{
if (m_frame && m_camera) {
Aurora::StreamCamera::CameraInfo info;
if (m_camera->getInfo(info)) {
if (m_frame->chromaStep == 1) {
auto result = YUVI420Rotate(
m_frame->y,
m_frame->cr,
m_frame->cb,
m_frame->width,
m_frame->height,
info.mountAngle
);
QBuffer qbuffer;
qbuffer.open(QIODevice::WriteOnly);
QSize size(result.width, result.height);
QImage image(size, QImage::Format_RGBA8888);
libyuv::I420ToARGB(
result.y, result.strideY,
result.u, result.strideU,
result.v, result.strideV,
reinterpret_cast<uint8_t *>(image.bits()),
result.width * 4,
result.width,
result.height
);
image.save(&qbuffer, "JPEG");
return qbuffer.data().toBase64().toStdString();
}
}
}
return "";
}
std::vector<Encodable> TextureCamera::GetAvailableCameras()
{
std::vector<Encodable> cameras;
@ -52,12 +93,12 @@ std::map<Encodable, Encodable> TextureCamera::GetState()
auto orientation = static_cast<int>(PlatformMethods::GetOrientation());
return std::map<Encodable, Encodable>{
return std::map<Encodable, Encodable> {
{"id", info.id},
{"textureId", m_textureId},
{"width", m_captureWidth},
{"height", m_captureHeight},
{"rotationCamera", info.mountAngle},
{"mountAngle", info.mountAngle},
{"rotationDisplay", orientation},
{"error", m_error},
};
@ -97,32 +138,18 @@ void TextureCamera::SendError(std::string error)
m_onError();
}
std::map<Encodable, Encodable> TextureCamera::StartCapture()
std::map<Encodable, Encodable> TextureCamera::StartCapture(size_t width, size_t height)
{
m_viewWidth = width;
m_viewHeight = height;
if (m_camera) {
if (m_camera->captureStarted()) {
m_camera->stopCapture();
}
Aurora::StreamCamera::CameraInfo info;
if (m_camera->getInfo(info)) {
std::vector<Aurora::StreamCamera::CameraCapability> caps;
if (m_manager->queryCapabilities(info.id, caps)) {
auto cap = caps.back();
// for(int i = caps.size()-1; i >= 0; i--) {
// if (displayWidth + displayHeight >= caps[i].width + caps[i].height) {
// cap = caps[i];
// break;
// }
// }
std::cout << "Version swscale_version: " << swscale_version() << std::endl;
m_captureWidth = cap.width;
m_captureHeight = cap.height;
ResizeFrame(width, height, info, cap);
if (!m_camera->startCapture(cap)) {
Unregister();
SendError("Stream camera error start capture");
@ -137,6 +164,7 @@ std::map<Encodable, Encodable> TextureCamera::StartCapture()
void TextureCamera::StopCapture()
{
if (m_camera && m_camera->captureStarted()) {
m_frame = nullptr;
m_camera->stopCapture();
}
}
@ -145,7 +173,7 @@ std::map<Encodable, Encodable> TextureCamera::Register(std::string cameraName)
{
m_textureId = m_plugin->RegisterTexture(
[this]([[maybe_unused]] size_t width, [[maybe_unused]] size_t height) -> std::optional<TextureVariant> {
if (m_bits) {
if (m_bits && m_captureWidth != 0 && m_captureHeight != 0) {
return std::make_optional(TextureVariant(FlutterPixelBuffer{
m_bits,
m_captureWidth,
@ -155,82 +183,129 @@ std::map<Encodable, Encodable> TextureCamera::Register(std::string cameraName)
return std::nullopt;
});
if (CreateCamera(cameraName)) {
StartCapture();
if (CreateCamera(cameraName) && m_viewWidth != 0) {
StartCapture(m_viewWidth, m_viewHeight);
}
return GetState();
}
std::map<Encodable, Encodable> TextureCamera::Unregister()
std::map<Encodable, Encodable> TextureCamera::Unregister()
{
StopCapture();
m_plugin->UnregisterTexture(m_textureId);
m_error = "";
m_counter = 0;
m_textureId = 0;
m_captureWidth = 0;
m_captureHeight = 0;
m_bits = nullptr;
m_camera = nullptr;
m_plugin->UnregisterTexture(m_textureId);
return GetState();
}
std::map<Encodable, Encodable> TextureCamera::ResizeFrame(size_t width, size_t height)
{
if (m_camera && m_camera->captureStarted() && !(width == m_captureWidth || height == m_captureHeight)) {
Aurora::StreamCamera::CameraInfo info;
if (m_camera->getInfo(info)) {
std::vector<Aurora::StreamCamera::CameraCapability> caps;
if (m_manager->queryCapabilities(info.id, caps)) {
auto cap = caps.back();
ResizeFrame(width, height, info, cap);
}
}
}
return GetState();
}
void TextureCamera::onCameraFrame(std::shared_ptr<Aurora::StreamCamera::GraphicBuffer> buffer)
void TextureCamera::ResizeFrame(size_t width, size_t height, Aurora::StreamCamera::CameraInfo info, Aurora::StreamCamera::CameraCapability cap)
{
auto cw = cap.width;
auto ch = cap.height;
auto dw = width < 500 ? 500 : width + 100;
auto dh = height < 500 ? 500 : height + 100;
if (info.mountAngle == 270 || info.mountAngle == 90) {
cw = cap.height;
ch = cap.width;
}
m_bits = nullptr;
m_captureHeight = dh;
m_captureWidth = (cw * dh) / ch;
if (m_captureWidth > dw) {
m_captureWidth = dw;
m_captureHeight = (ch * dw) / cw;
}
}
std::optional<std::shared_ptr<const Aurora::StreamCamera::YCbCrFrame>> TextureCamera::GetFrame(std::shared_ptr<Aurora::StreamCamera::GraphicBuffer> buffer)
{
m_counter += 1;
if (m_counter < 0) m_counter = 0;
if (m_counter %3 == 0) return;
auto frame = buffer->mapYCbCr();
unsigned int displayWidth = PlatformMethods::GetDisplayWidth();
unsigned int displayHeight = PlatformMethods::GetDisplayHeight();
auto bits = std::shared_ptr<uint8_t>((uint8_t *) malloc(frame->width * 4 * frame->height), free);
auto yPtr = std::shared_ptr<uint8_t>((uint8_t *) malloc(frame->width * 4 * frame->height), free);
auto uPtr = std::shared_ptr<uint8_t>((uint8_t *) malloc(frame->width * 4 * frame->height), free);
auto vPtr = std::shared_ptr<uint8_t>((uint8_t *) malloc(frame->width * 4 * frame->height), free);
libyuv::I420Scale(
frame->y,
frame->yStride,
frame->cr,
frame->cStride,
frame->cb,
frame->cStride,
frame->width,
frame->height,
yPtr.get(),
frame->yStride,
uPtr.get(),
frame->cStride,
vPtr.get(),
frame->cStride,
frame->width,
frame->height,
libyuv::kFilterBilinear
);
libyuv::Android420ToARGB(
yPtr.get(),
frame->yStride,
uPtr.get(),
frame->cStride,
vPtr.get(),
frame->cStride,
frame->chromaStep,
bits.get(),
frame->width * 4,
frame->width,
frame->height
);
if (m_counter < 0) {
m_counter = 0;
}
if (m_counter %3 == 0 || !m_camera) {
return std::nullopt;
}
m_frame = buffer->mapYCbCr();
return m_frame;
}
void TextureCamera::onCameraFrame(std::shared_ptr<Aurora::StreamCamera::GraphicBuffer> buffer)
{
if (auto optional = GetFrame(buffer)) {
auto frame = optional.value();
if (!m_camera || !m_camera->captureStarted()) {
return;
}
auto result = YUVI420Scale(
frame->y,
frame->cr,
frame->cb,
frame->width,
frame->height,
m_captureWidth,
m_captureHeight
);
if (!m_camera || !m_camera->captureStarted()) {
return;
}
auto bits = std::shared_ptr<uint8_t>((uint8_t *) malloc(result.width * result.height * 4), free);
m_bits = bits;
m_plugin->MarkTextureAvailable(m_textureId);
libyuv::I420ToARGB(
result.y, result.strideY,
result.u, result.strideU,
result.v, result.strideV,
bits.get(),
result.width * 4,
result.width,
result.height
);
if (!m_camera || !m_camera->captureStarted()) {
return;
}
m_bits = bits;
m_plugin->MarkTextureAvailable(m_textureId);
}
}
void TextureCamera::onCameraError(const std::string &errorDescription)
@ -244,3 +319,108 @@ void TextureCamera::onCameraParameterChanged([[maybe_unused]] Aurora::StreamCame
{
std::cout << "onCameraParameterChanged: " << value << std::endl;
}
ResultYUV TextureCamera::YUVI420Scale(
const uint8_t* srcY,
const uint8_t* srcU,
const uint8_t* srcV,
int srcWidth,
int srcHeight,
int outWidth,
int outHeight
)
{
auto bufSize = (((outWidth * outHeight) + ((outWidth + 1) / 2) * ((outHeight + 1) / 2))) * 2;
auto buf = std::shared_ptr<uint8_t>((uint8_t *) malloc(bufSize), free);
auto y = buf.get();
auto u = buf.get() + outWidth * outHeight;
auto v = buf.get() + outWidth * outHeight + (outWidth * outHeight + 3) / 4;
auto srcStrideY = srcWidth;
auto srcStrideU = (srcWidth + 1) / 2;
auto srcStrideV = srcStrideU;
auto outStrideY = outWidth;
auto outStrideU = (outWidth + 1) / 2;
auto outStrideV = outStrideU;
libyuv::I420Scale(
srcY, srcStrideY,
srcU, srcStrideU,
srcV, srcStrideV,
srcWidth,
srcHeight,
y, outStrideY,
u, outStrideU,
v, outStrideV,
outWidth,
outHeight,
libyuv::kFilterBilinear
);
return ResultYUV{
y, outStrideY,
u, outStrideU,
v, outStrideV,
outWidth,
outHeight,
buf
};
}
ResultYUV TextureCamera::YUVI420Rotate(
const uint8_t* srcY,
const uint8_t* srcU,
const uint8_t* srcV,
int srcWidth,
int srcHeight,
int degree // 0, 90, 180, 270
)
{
int outWidth = srcWidth;
int outHeight = srcHeight;
if (degree == 90 || degree == 270) {
outWidth = srcHeight;
outHeight = srcWidth;
}
enum libyuv::RotationMode mode = (enum libyuv::RotationMode) degree;
auto bufSize = (((outWidth * outHeight) + ((outWidth + 1) / 2) * ((outHeight + 1) / 2))) * 2;
auto buf = std::shared_ptr<uint8_t>((uint8_t *) malloc(bufSize), free);
auto y = buf.get();
auto u = buf.get() + outWidth * outHeight;
auto v = buf.get() + outWidth * outHeight + (outWidth * outHeight + 3) / 4;
auto srcStrideY = srcWidth;
auto srcStrideU = (srcWidth + 1) / 2;
auto srcStrideV = srcStrideU;
auto outStrideY = outWidth;
auto outStrideU = (outWidth + 1) / 2;
auto outStrideV = outStrideU;
libyuv::I420Rotate(
srcY, srcStrideY,
srcU, srcStrideU,
srcV, srcStrideV,
y, outStrideY,
u, outStrideU,
v, outStrideV,
srcWidth,
srcHeight,
mode
);
return ResultYUV{
y, outStrideY,
u, outStrideU,
v, outStrideV,
outWidth,
outHeight,
buf
};
}

6
packages/camera/camera_aurora/lib/camera_aurora.dart

@ -37,7 +37,9 @@ class CameraAurora extends CameraPlatform {
}
@override
Future<void> dispose(int cameraId) => CameraAuroraPlatform.instance.dispose();
Future<void> dispose(int cameraId) async {
await CameraAuroraPlatform.instance.dispose();
}
@override
Future<XFile> takePicture(int cameraId) =>
@ -85,6 +87,8 @@ class CameraAurora extends CameraPlatform {
BuildContext context,
BoxConstraints constraints,
) {
CameraAuroraPlatform.instance
.resizeFrame(constraints.maxWidth, constraints.maxHeight);
return CameraViewfinder(
width: constraints.maxWidth,
height: constraints.maxHeight,

15
packages/camera/camera_aurora/lib/camera_aurora_method_channel.dart

@ -12,6 +12,7 @@ import 'camera_data.dart';
enum CameraAuroraMethods {
availableCameras,
createCamera,
resizeFrame,
dispose,
startCapture,
stopCapture,
@ -78,6 +79,15 @@ class MethodChannelCameraAurora extends CameraAuroraPlatform {
return CameraState.fromJson(data ?? {});
}
@override
Future<void> resizeFrame(double width, double height) async {
await methodsChannel
.invokeMethod<Object?>(CameraAuroraMethods.resizeFrame.name, {
'width': width.round(),
'height': height.round(),
});
}
@override
Future<void> startCapture(double width, double height) async {
await methodsChannel
@ -105,7 +115,10 @@ class MethodChannelCameraAurora extends CameraAuroraPlatform {
CameraAuroraMethods.takePicture.name,
{'cameraId': cameraId},
);
final bytes = base64Decode(image!);
if (image!.isEmpty) {
throw CameraException('nv12', 'Empty image data!');
}
final bytes = base64Decode(image);
return XFile.fromData(
bytes,
name: 'temp.jpg',

4
packages/camera/camera_aurora/lib/camera_aurora_platform_interface.dart

@ -48,6 +48,10 @@ abstract class CameraAuroraPlatform extends PlatformInterface {
throw UnimplementedError('createCamera() has not been implemented.');
}
Future<void> resizeFrame(double width, double height) {
throw UnimplementedError('resizeFrame() has not been implemented.');
}
Future<void> dispose() {
throw UnimplementedError('dispose() has not been implemented.');
}

6
packages/camera/camera_aurora/lib/camera_data.dart

@ -15,7 +15,7 @@ class CameraState {
textureId = json['textureId'] ?? -1,
width = (json['width'] ?? 0).toDouble(),
height = (json['height'] ?? 0).toDouble(),
rotationCamera = json['rotationCamera'] ?? 0,
mountAngle = json['mountAngle'] ?? 0,
rotationDisplay = json['rotationDisplay'] ?? 0,
error = json['error'] ?? '';
@ -23,7 +23,7 @@ class CameraState {
final int textureId;
final double width;
final double height;
final int rotationCamera;
final int mountAngle;
final int rotationDisplay;
final String error;
@ -33,6 +33,6 @@ class CameraState {
@override
String toString() {
return '{id: $id, textureId: $textureId, width: $width, height: $height, rotationCamera: $rotationCamera, rotationDisplay: $rotationDisplay, error: $error}';
return '{id: $id, textureId: $textureId, width: $width, height: $height, mountAngle: $mountAngle, rotationDisplay: $rotationDisplay, error: $error}';
}
}

61
packages/camera/camera_aurora/lib/camera_viewfinder.dart

@ -37,8 +37,8 @@ class _CameraViewfinderState extends State<CameraViewfinder> {
@override
void dispose() {
super.dispose();
CameraAuroraPlatform.instance.stopCapture();
super.dispose();
}
@override
@ -54,43 +54,54 @@ class _CameraViewfinderState extends State<CameraViewfinder> {
} else if (_cameraState.isNotEmpty()) {
int turn = 0;
// @todo Different direction of rotation
bool isFront = _cameraState.id.contains('front');
switch (_cameraState.mountAngle) {
case 0:
turn += 0;
break;
case 90:
turn += 1;
break;
case 180:
turn += 2;
break;
default: // 270
turn += 3;
}
switch (_cameraState.rotationDisplay) {
case 0:
turn = _cameraState.id.contains('front') ? -1 : 1;
turn += 0;
break;
case 90:
turn = 0;
turn += isFront ? 1 : -1;
break;
case 180:
turn = _cameraState.id.contains('front') ? 1 : -1;
turn += 2;
break;
default: // 270
turn = 2;
turn += isFront ? 3 : -3;
}
double height = 10;
double width = 10;
double cw = turn % 2 == 0 ? _cameraState.height : _cameraState.width;
double ch = turn % 2 == 0 ? _cameraState.width : _cameraState.height;
double height = widget.height;
double width = widget.height * cw / ch;
if (_cameraState.height != 0 && _cameraState.width != 0) {
if (_cameraState.rotationDisplay == 90 ||
_cameraState.rotationDisplay == 270) {
width = widget.width * _cameraState.height / _cameraState.width;
height = widget.height * _cameraState.width / _cameraState.height;
} else {
width = _cameraState.height * widget.height / _cameraState.width;
height = _cameraState.width * widget.width / _cameraState.height;
}
if (width > widget.width) {
width = widget.width;
height = widget.width * ch / cw;
}
return RotatedBox(
quarterTurns: turn,
child: SizedBox(
width: height, // height
height: width, // widht
child: Opacity(
opacity: _cameraState.height == 0 ? 0 : 1,
child: Texture(textureId: _cameraState.textureId),
),
return SizedBox(
width: width,
height: height,
child: RotatedBox(
quarterTurns: turn,
child: Texture(textureId: _cameraState.textureId),
),
);
}

Loading…
Cancel
Save