/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "mozilla/dom/ImageBitmap.h" #include "mozilla/CheckedInt.h" #include "mozilla/dom/File.h" // for Blob #include "mozilla/dom/HTMLImageElement.h" #include "mozilla/dom/ImageBitmapBinding.h" #include "mozilla/dom/Promise.h" #include "mozilla/dom/StructuredCloneTags.h" #include "mozilla/dom/UnionTypes.h" // for ArrayBufferViewOrArrayBuffer #include "mozilla/dom/WorkerPrivate.h" #include "mozilla/dom/WorkerRunnable.h" #include "CanvasRenderingContext2D.h" #include "gfxUtils.h" #include "ImageBitmapColorUtils.h" #include "ImageBitmapUtils.h" #include "ImageData.h" #include "ImageLayers.h" #include "ImageUtils.h" #include "imgTools.h" #include "libyuv.h" #include "nsLayoutUtils.h" #include "OffscreenCanvas.h" using namespace mozilla::dom; using namespace mozilla::gfx; using namespace mozilla::layers; namespace mozilla { namespace dom { using namespace workers; NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(ImageBitmap, mParent) NS_IMPL_CYCLE_COLLECTING_ADDREF(ImageBitmap) NS_IMPL_CYCLE_COLLECTING_RELEASE(ImageBitmap) NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(ImageBitmap) NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY NS_INTERFACE_MAP_ENTRY(nsISupports) NS_INTERFACE_MAP_END /* * If either aRect.width or aRect.height are negative, then return a new IntRect * which represents the same rectangle as the aRect does but with positive width * and height. */ static IntRect FixUpNegativeDimension(const IntRect& aRect, ErrorResult& aRv) { gfx::IntRect rect = aRect; // fix up negative dimensions if (rect.width < 0) { CheckedInt32 checkedX = CheckedInt32(rect.x) + rect.width; if (!checkedX.isValid()) { aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); return rect; } rect.x = checkedX.value(); rect.width = -(rect.width); } if (rect.height < 0) { CheckedInt32 checkedY = CheckedInt32(rect.y) + rect.height; if (!checkedY.isValid()) { aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); return rect; } rect.y = checkedY.value(); rect.height = -(rect.height); } return rect; } /* * This helper function copies the data of the given DataSourceSurface, * _aSurface_, in the given area, _aCropRect_, into a new DataSourceSurface. * This might return null if it can not create a new SourceSurface or it cannot * read data from the given _aSurface_. * * Warning: Even though the area of _aCropRect_ is just the same as the size of * _aSurface_, this function still copy data into a new * DataSourceSurface. */ static already_AddRefed CropAndCopyDataSourceSurface(DataSourceSurface* aSurface, const IntRect& aCropRect) { MOZ_ASSERT(aSurface); // Check the aCropRect ErrorResult error; const IntRect positiveCropRect = FixUpNegativeDimension(aCropRect, error); if (NS_WARN_IF(error.Failed())) { error.SuppressException(); return nullptr; } // Calculate the size of the new SourceSurface. // We cannot keep using aSurface->GetFormat() to create new DataSourceSurface, // since it might be SurfaceFormat::B8G8R8X8 which does not handle opacity, // however the specification explicitly define that "If any of the pixels on // this rectangle are outside the area where the input bitmap was placed, then // they will be transparent black in output." // So, instead, we force the output format to be SurfaceFormat::B8G8R8A8. const SurfaceFormat format = SurfaceFormat::B8G8R8A8; const int bytesPerPixel = BytesPerPixel(format); const IntSize dstSize = IntSize(positiveCropRect.width, positiveCropRect.height); const uint32_t dstStride = dstSize.width * bytesPerPixel; // Create a new SourceSurface. RefPtr dstDataSurface = Factory::CreateDataSourceSurfaceWithStride(dstSize, format, dstStride, true); if (NS_WARN_IF(!dstDataSurface)) { return nullptr; } // Only do copying and cropping when the positiveCropRect intersects with // the size of aSurface. const IntRect surfRect(IntPoint(0, 0), aSurface->GetSize()); if (surfRect.Intersects(positiveCropRect)) { const IntRect surfPortion = surfRect.Intersect(positiveCropRect); const IntPoint dest(std::max(0, surfPortion.X() - positiveCropRect.X()), std::max(0, surfPortion.Y() - positiveCropRect.Y())); // Copy the raw data into the newly created DataSourceSurface. DataSourceSurface::ScopedMap srcMap(aSurface, DataSourceSurface::READ); DataSourceSurface::ScopedMap dstMap(dstDataSurface, DataSourceSurface::WRITE); if (NS_WARN_IF(!srcMap.IsMapped()) || NS_WARN_IF(!dstMap.IsMapped())) { return nullptr; } uint8_t* srcBufferPtr = srcMap.GetData() + surfPortion.y * srcMap.GetStride() + surfPortion.x * bytesPerPixel; uint8_t* dstBufferPtr = dstMap.GetData() + dest.y * dstMap.GetStride() + dest.x * bytesPerPixel; CheckedInt copiedBytesPerRaw = CheckedInt(surfPortion.width) * bytesPerPixel; if (!copiedBytesPerRaw.isValid()) { return nullptr; } for (int i = 0; i < surfPortion.height; ++i) { memcpy(dstBufferPtr, srcBufferPtr, copiedBytesPerRaw.value()); srcBufferPtr += srcMap.GetStride(); dstBufferPtr += dstMap.GetStride(); } } return dstDataSurface.forget(); } /* * Encapsulate the given _aSurface_ into a layers::SourceSurfaceImage. */ static already_AddRefed CreateImageFromSurface(SourceSurface* aSurface) { MOZ_ASSERT(aSurface); RefPtr image = new layers::SourceSurfaceImage(aSurface->GetSize(), aSurface); return image.forget(); } /* * CreateImageFromRawData(), CreateSurfaceFromRawData() and * CreateImageFromRawDataInMainThreadSyncTask are helpers for * create-from-ImageData case */ static already_AddRefed CreateSurfaceFromRawData(const gfx::IntSize& aSize, uint32_t aStride, gfx::SurfaceFormat aFormat, uint8_t* aBuffer, uint32_t aBufferLength, const Maybe& aCropRect) { MOZ_ASSERT(!aSize.IsEmpty()); MOZ_ASSERT(aBuffer); // Wrap the source buffer into a SourceSurface. RefPtr dataSurface = Factory::CreateWrappingDataSourceSurface(aBuffer, aStride, aSize, aFormat); if (NS_WARN_IF(!dataSurface)) { return nullptr; } // The temporary cropRect variable is equal to the size of source buffer if we // do not need to crop, or it equals to the given cropping size. const IntRect cropRect = aCropRect.valueOr(IntRect(0, 0, aSize.width, aSize.height)); // Copy the source buffer in the _cropRect_ area into a new SourceSurface. RefPtr result = CropAndCopyDataSourceSurface(dataSurface, cropRect); if (NS_WARN_IF(!result)) { return nullptr; } return result.forget(); } static already_AddRefed CreateImageFromRawData(const gfx::IntSize& aSize, uint32_t aStride, gfx::SurfaceFormat aFormat, uint8_t* aBuffer, uint32_t aBufferLength, const Maybe& aCropRect) { MOZ_ASSERT(NS_IsMainThread()); // Copy and crop the source buffer into a SourceSurface. RefPtr rgbaSurface = CreateSurfaceFromRawData(aSize, aStride, aFormat, aBuffer, aBufferLength, aCropRect); if (NS_WARN_IF(!rgbaSurface)) { return nullptr; } // Convert RGBA to BGRA RefPtr rgbaDataSurface = rgbaSurface->GetDataSurface(); RefPtr bgraDataSurface = Factory::CreateDataSourceSurfaceWithStride(rgbaDataSurface->GetSize(), SurfaceFormat::B8G8R8A8, rgbaDataSurface->Stride()); DataSourceSurface::MappedSurface rgbaMap; DataSourceSurface::MappedSurface bgraMap; if (NS_WARN_IF(!rgbaDataSurface->Map(DataSourceSurface::MapType::READ, &rgbaMap)) || NS_WARN_IF(!bgraDataSurface->Map(DataSourceSurface::MapType::WRITE, &bgraMap))) { return nullptr; } libyuv::ABGRToARGB(rgbaMap.mData, rgbaMap.mStride, bgraMap.mData, bgraMap.mStride, bgraDataSurface->GetSize().width, bgraDataSurface->GetSize().height); rgbaDataSurface->Unmap(); bgraDataSurface->Unmap(); // Create an Image from the BGRA SourceSurface. RefPtr image = CreateImageFromSurface(bgraDataSurface); if (NS_WARN_IF(!image)) { return nullptr; } return image.forget(); } /* * This is a synchronous task. * This class is used to create a layers::SourceSurfaceImage from raw data in the main * thread. While creating an ImageBitmap from an ImageData, we need to create * a SouceSurface from the ImageData's raw data and then set the SourceSurface * into a layers::SourceSurfaceImage. However, the layers::SourceSurfaceImage asserts the * setting operation in the main thread, so if we are going to create an * ImageBitmap from an ImageData off the main thread, we post an event to the * main thread to create a layers::SourceSurfaceImage from an ImageData's raw data. */ class CreateImageFromRawDataInMainThreadSyncTask final : public WorkerMainThreadRunnable { public: CreateImageFromRawDataInMainThreadSyncTask(uint8_t* aBuffer, uint32_t aBufferLength, uint32_t aStride, gfx::SurfaceFormat aFormat, const gfx::IntSize& aSize, const Maybe& aCropRect, layers::Image** aImage) : WorkerMainThreadRunnable(GetCurrentThreadWorkerPrivate(), NS_LITERAL_CSTRING("ImageBitmap :: Create Image from Raw Data")) , mImage(aImage) , mBuffer(aBuffer) , mBufferLength(aBufferLength) , mStride(aStride) , mFormat(aFormat) , mSize(aSize) , mCropRect(aCropRect) { MOZ_ASSERT(!(*aImage), "Don't pass an existing Image into CreateImageFromRawDataInMainThreadSyncTask."); } bool MainThreadRun() override { RefPtr image = CreateImageFromRawData(mSize, mStride, mFormat, mBuffer, mBufferLength, mCropRect); if (NS_WARN_IF(!image)) { return false; } image.forget(mImage); return true; } private: layers::Image** mImage; uint8_t* mBuffer; uint32_t mBufferLength; uint32_t mStride; gfx::SurfaceFormat mFormat; gfx::IntSize mSize; const Maybe& mCropRect; }; static bool CheckSecurityForHTMLElements(bool aIsWriteOnly, bool aCORSUsed, nsIPrincipal* aPrincipal) { MOZ_ASSERT(aPrincipal); if (aIsWriteOnly) { return false; } if (!aCORSUsed) { nsIGlobalObject* incumbentSettingsObject = GetIncumbentGlobal(); if (NS_WARN_IF(!incumbentSettingsObject)) { return false; } nsIPrincipal* principal = incumbentSettingsObject->PrincipalOrNull(); if (NS_WARN_IF(!principal) || !(principal->Subsumes(aPrincipal))) { return false; } } return true; } static bool CheckSecurityForHTMLElements(const nsLayoutUtils::SurfaceFromElementResult& aRes) { return CheckSecurityForHTMLElements(aRes.mIsWriteOnly, aRes.mCORSUsed, aRes.mPrincipal); } /* * A wrapper to the nsLayoutUtils::SurfaceFromElement() function followed by the * security checking. */ template static already_AddRefed GetSurfaceFromElement(nsIGlobalObject* aGlobal, HTMLElementType& aElement, bool* aWriteOnly, ErrorResult& aRv) { nsLayoutUtils::SurfaceFromElementResult res = nsLayoutUtils::SurfaceFromElement(&aElement, nsLayoutUtils::SFE_WANT_FIRST_FRAME); RefPtr surface = res.GetSourceSurface(); if (NS_WARN_IF(!surface)) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } // Check origin-clean and pass back *aWriteOnly = !CheckSecurityForHTMLElements(res); return surface.forget(); } /* * The specification doesn't allow to create an ImageBitmap from a vector image. * This function is used to check if the given HTMLImageElement contains a * raster image. */ static bool HasRasterImage(HTMLImageElement& aImageEl) { nsresult rv; nsCOMPtr imgRequest; rv = aImageEl.GetRequest(nsIImageLoadingContent::CURRENT_REQUEST, getter_AddRefs(imgRequest)); if (NS_SUCCEEDED(rv) && imgRequest) { nsCOMPtr imgContainer; rv = imgRequest->GetImage(getter_AddRefs(imgContainer)); if (NS_SUCCEEDED(rv) && imgContainer && imgContainer->GetType() == imgIContainer::TYPE_RASTER) { return true; } } return false; } ImageBitmap::ImageBitmap(nsIGlobalObject* aGlobal, layers::Image* aData, bool aWriteOnly, bool aIsPremultipliedAlpha /* = true */) : mParent(aGlobal) , mData(aData) , mSurface(nullptr) , mDataWrapper(new ImageUtils(mData)) , mPictureRect(0, 0, aData->GetSize().width, aData->GetSize().height) , mIsPremultipliedAlpha(aIsPremultipliedAlpha) , mIsCroppingAreaOutSideOfSourceImage(false) , mWriteOnly(aWriteOnly) { MOZ_ASSERT(aData, "aData is null in ImageBitmap constructor."); } ImageBitmap::~ImageBitmap() { } JSObject* ImageBitmap::WrapObject(JSContext* aCx, JS::Handle aGivenProto) { return ImageBitmapBinding::Wrap(aCx, this, aGivenProto); } void ImageBitmap::Close() { mData = nullptr; mSurface = nullptr; mPictureRect.SetEmpty(); } void ImageBitmap::SetPictureRect(const IntRect& aRect, ErrorResult& aRv) { mPictureRect = FixUpNegativeDimension(aRect, aRv); } void ImageBitmap::SetIsCroppingAreaOutSideOfSourceImage(const IntSize& aSourceSize, const Maybe& aCroppingRect) { // No cropping at all. if (aCroppingRect.isNothing()) { mIsCroppingAreaOutSideOfSourceImage = false; return; } if (aCroppingRect->X() < 0 || aCroppingRect->Y() < 0 || aCroppingRect->Width() > aSourceSize.width || aCroppingRect->Height() > aSourceSize.height) { mIsCroppingAreaOutSideOfSourceImage = true; } } static already_AddRefed ConvertColorFormatIfNeeded(RefPtr aSurface) { const SurfaceFormat srcFormat = aSurface->GetFormat(); if (srcFormat == SurfaceFormat::R8G8B8A8 || srcFormat == SurfaceFormat::B8G8R8A8 || srcFormat == SurfaceFormat::R8G8B8X8 || srcFormat == SurfaceFormat::B8G8R8X8 || srcFormat == SurfaceFormat::A8R8G8B8 || srcFormat == SurfaceFormat::X8R8G8B8) { return aSurface.forget(); } if (srcFormat == SurfaceFormat::A8 || srcFormat == SurfaceFormat::Depth) { return nullptr; } const int bytesPerPixel = BytesPerPixel(SurfaceFormat::B8G8R8A8); const IntSize dstSize = aSurface->GetSize(); const uint32_t dstStride = dstSize.width * bytesPerPixel; RefPtr dstDataSurface = Factory::CreateDataSourceSurfaceWithStride(dstSize, SurfaceFormat::B8G8R8A8, dstStride); RefPtr srcDataSurface = aSurface->GetDataSurface(); if (NS_WARN_IF(!srcDataSurface)) { return nullptr; } DataSourceSurface::ScopedMap srcMap(srcDataSurface, DataSourceSurface::READ); DataSourceSurface::ScopedMap dstMap(dstDataSurface, DataSourceSurface::WRITE); if (NS_WARN_IF(!srcMap.IsMapped()) || NS_WARN_IF(!dstMap.IsMapped())) { return nullptr; } int rv = 0; if (srcFormat == SurfaceFormat::R8G8B8) { rv = RGB24ToBGRA32(srcMap.GetData(), srcMap.GetStride(), dstMap.GetData(), dstMap.GetStride(), dstSize.width, dstSize.height); } else if (srcFormat == SurfaceFormat::B8G8R8) { rv = BGR24ToBGRA32(srcMap.GetData(), srcMap.GetStride(), dstMap.GetData(), dstMap.GetStride(), dstSize.width, dstSize.height); } else if (srcFormat == SurfaceFormat::HSV) { rv = HSVToBGRA32((const float*)srcMap.GetData(), srcMap.GetStride(), dstMap.GetData(), dstMap.GetStride(), dstSize.width, dstSize.height); } else if (srcFormat == SurfaceFormat::Lab) { rv = LabToBGRA32((const float*)srcMap.GetData(), srcMap.GetStride(), dstMap.GetData(), dstMap.GetStride(), dstSize.width, dstSize.height); } if (NS_WARN_IF(rv != 0)) { return nullptr; } return dstDataSurface.forget(); } /* * The functionality of PrepareForDrawTarget method: * (1) Get a SourceSurface from the mData (which is a layers::Image). * (2) Convert the SourceSurface to format B8G8R8A8 if the original format is * R8G8B8, B8G8R8, HSV or Lab. * Note: if the original format is A8 or Depth, then return null directly. * (3) Do cropping if the size of SourceSurface does not equal to the * mPictureRect. * (4) Pre-multiply alpha if needed. */ already_AddRefed ImageBitmap::PrepareForDrawTarget(gfx::DrawTarget* aTarget) { MOZ_ASSERT(aTarget); if (!mData) { return nullptr; } if (!mSurface) { mSurface = mData->GetAsSourceSurface(); if (!mSurface) { return nullptr; } } // Check if we need to convert the format. // Convert R8G8B8/B8G8R8/HSV/Lab to B8G8R8A8. // Return null if the original format is A8 or Depth. mSurface = ConvertColorFormatIfNeeded(mSurface); if (NS_WARN_IF(!mSurface)) { return nullptr; } RefPtr target = aTarget; IntRect surfRect(0, 0, mSurface->GetSize().width, mSurface->GetSize().height); // Check if we still need to crop our surface if (!mPictureRect.IsEqualEdges(surfRect)) { IntRect surfPortion = surfRect.Intersect(mPictureRect); // the crop lies entirely outside the surface area, nothing to draw if (surfPortion.IsEmpty()) { mSurface = nullptr; RefPtr surface(mSurface); return surface.forget(); } IntPoint dest(std::max(0, surfPortion.X() - mPictureRect.X()), std::max(0, surfPortion.Y() - mPictureRect.Y())); // We must initialize this target with mPictureRect.Size() because the // specification states that if the cropping area is given, then return an // ImageBitmap with the size equals to the cropping area. target = target->CreateSimilarDrawTarget(mPictureRect.Size(), target->GetFormat()); if (!target) { mSurface = nullptr; RefPtr surface(mSurface); return surface.forget(); } target->CopySurface(mSurface, surfPortion, dest); mSurface = target->Snapshot(); // Make mCropRect match new surface we've cropped to mPictureRect.MoveTo(0, 0); } // Pre-multiply alpha here. // Apply pre-multiply alpha only if mIsPremultipliedAlpha is false. // Ignore this step if the source surface does not have alpha channel; this // kind of source surfaces might come form layers::PlanarYCbCrImage. if (!mIsPremultipliedAlpha && mSurface->GetFormat() != SurfaceFormat::B8G8R8X8 && mSurface->GetFormat() != SurfaceFormat::R8G8B8X8 && mSurface->GetFormat() != SurfaceFormat::X8R8G8B8) { MOZ_ASSERT(mSurface->GetFormat() == SurfaceFormat::R8G8B8A8 || mSurface->GetFormat() == SurfaceFormat::B8G8R8A8 || mSurface->GetFormat() == SurfaceFormat::A8R8G8B8); RefPtr dstSurface = mSurface->GetDataSurface(); MOZ_ASSERT(dstSurface); RefPtr srcSurface; DataSourceSurface::MappedSurface srcMap; DataSourceSurface::MappedSurface dstMap; if (!dstSurface->Map(DataSourceSurface::MapType::READ_WRITE, &dstMap)) { srcSurface = dstSurface; if (!srcSurface->Map(DataSourceSurface::READ, &srcMap)) { gfxCriticalError() << "Failed to map source surface for premultiplying alpha."; return nullptr; } dstSurface = Factory::CreateDataSourceSurface(srcSurface->GetSize(), srcSurface->GetFormat()); if (!dstSurface || !dstSurface->Map(DataSourceSurface::MapType::WRITE, &dstMap)) { gfxCriticalError() << "Failed to map destination surface for premultiplying alpha."; srcSurface->Unmap(); return nullptr; } } uint8_t rIndex = 0; uint8_t gIndex = 0; uint8_t bIndex = 0; uint8_t aIndex = 0; if (mSurface->GetFormat() == SurfaceFormat::R8G8B8A8) { rIndex = 0; gIndex = 1; bIndex = 2; aIndex = 3; } else if (mSurface->GetFormat() == SurfaceFormat::B8G8R8A8) { rIndex = 2; gIndex = 1; bIndex = 0; aIndex = 3; } else if (mSurface->GetFormat() == SurfaceFormat::A8R8G8B8) { rIndex = 1; gIndex = 2; bIndex = 3; aIndex = 0; } for (int i = 0; i < dstSurface->GetSize().height; ++i) { uint8_t* bufferPtr = dstMap.mData + dstMap.mStride * i; uint8_t* srcBufferPtr = srcSurface ? srcMap.mData + srcMap.mStride * i : bufferPtr; for (int i = 0; i < dstSurface->GetSize().width; ++i) { uint8_t r = *(srcBufferPtr+rIndex); uint8_t g = *(srcBufferPtr+gIndex); uint8_t b = *(srcBufferPtr+bIndex); uint8_t a = *(srcBufferPtr+aIndex); *(bufferPtr+rIndex) = gfxUtils::sPremultiplyTable[a * 256 + r]; *(bufferPtr+gIndex) = gfxUtils::sPremultiplyTable[a * 256 + g]; *(bufferPtr+bIndex) = gfxUtils::sPremultiplyTable[a * 256 + b]; *(bufferPtr+aIndex) = a; bufferPtr += 4; srcBufferPtr += 4; } } dstSurface->Unmap(); if (srcSurface) { srcSurface->Unmap(); } mSurface = dstSurface; } // Replace our surface with one optimized for the target we're about to draw // to, under the assumption it'll likely be drawn again to that target. // This call should be a no-op for already-optimized surfaces mSurface = target->OptimizeSourceSurface(mSurface); RefPtr surface(mSurface); return surface.forget(); } already_AddRefed ImageBitmap::TransferAsImage() { RefPtr image = mData; Close(); return image.forget(); } UniquePtr ImageBitmap::ToCloneData() const { UniquePtr result(new ImageBitmapCloneData()); result->mPictureRect = mPictureRect; result->mIsPremultipliedAlpha = mIsPremultipliedAlpha; result->mIsCroppingAreaOutSideOfSourceImage = mIsCroppingAreaOutSideOfSourceImage; RefPtr surface = mData->GetAsSourceSurface(); result->mSurface = surface->GetDataSurface(); MOZ_ASSERT(result->mSurface); result->mWriteOnly = mWriteOnly; return Move(result); } /* static */ already_AddRefed ImageBitmap::CreateFromCloneData(nsIGlobalObject* aGlobal, ImageBitmapCloneData* aData) { RefPtr data = CreateImageFromSurface(aData->mSurface); RefPtr ret = new ImageBitmap(aGlobal, data, aData->mWriteOnly, aData->mIsPremultipliedAlpha); ret->mIsCroppingAreaOutSideOfSourceImage = aData->mIsCroppingAreaOutSideOfSourceImage; ErrorResult rv; ret->SetPictureRect(aData->mPictureRect, rv); return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateFromOffscreenCanvas(nsIGlobalObject* aGlobal, OffscreenCanvas& aOffscreenCanvas, ErrorResult& aRv) { // Check origin-clean bool writeOnly = aOffscreenCanvas.IsWriteOnly(); nsLayoutUtils::SurfaceFromElementResult res = nsLayoutUtils::SurfaceFromOffscreenCanvas(&aOffscreenCanvas, nsLayoutUtils::SFE_WANT_FIRST_FRAME); RefPtr surface = res.GetSourceSurface(); if (NS_WARN_IF(!surface)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } RefPtr data = CreateImageFromSurface(surface); RefPtr ret = new ImageBitmap(aGlobal, data, writeOnly); return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateInternal(nsIGlobalObject* aGlobal, HTMLImageElement& aImageEl, const Maybe& aCropRect, ErrorResult& aRv) { // Check if the image element is completely available or not. if (!aImageEl.Complete()) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } // Check if the image element is a bitmap (e.g. it's a vector graphic) or not. if (!HasRasterImage(aImageEl)) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } bool writeOnly = true; // Get the SourceSurface out from the image element and then do security // checking. RefPtr surface = GetSurfaceFromElement(aGlobal, aImageEl, &writeOnly, aRv); if (NS_WARN_IF(aRv.Failed())) { return nullptr; } // Create ImageBitmap. RefPtr data = CreateImageFromSurface(surface); if (NS_WARN_IF(!data)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } RefPtr ret = new ImageBitmap(aGlobal, data, writeOnly); // Set the picture rectangle. if (ret && aCropRect.isSome()) { ret->SetPictureRect(aCropRect.ref(), aRv); } // Set mIsCroppingAreaOutSideOfSourceImage. ret->SetIsCroppingAreaOutSideOfSourceImage(surface->GetSize(), aCropRect); return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateInternal(nsIGlobalObject* aGlobal, HTMLVideoElement& aVideoEl, const Maybe& aCropRect, ErrorResult& aRv) { aVideoEl.MarkAsContentSource(mozilla::dom::HTMLVideoElement::CallerAPI::CREATE_IMAGEBITMAP); // Check network state. if (aVideoEl.NetworkState() == HTMLMediaElement::NETWORK_EMPTY) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } // Check ready state. // Cannot be HTMLMediaElement::HAVE_NOTHING or HTMLMediaElement::HAVE_METADATA. if (aVideoEl.ReadyState() <= HTMLMediaElement::HAVE_METADATA) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } bool writeOnly = true; // Check security. nsCOMPtr principal = aVideoEl.GetCurrentVideoPrincipal(); bool CORSUsed = aVideoEl.GetCORSMode() != CORS_NONE; writeOnly = !CheckSecurityForHTMLElements(false, CORSUsed, principal); // Create ImageBitmap. ImageContainer *container = aVideoEl.GetImageContainer(); if (!container) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } AutoLockImage lockImage(container); layers::Image* data = lockImage.GetImage(); if (!data) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } RefPtr ret = new ImageBitmap(aGlobal, data, writeOnly); // Set the picture rectangle. if (ret && aCropRect.isSome()) { ret->SetPictureRect(aCropRect.ref(), aRv); } // Set mIsCroppingAreaOutSideOfSourceImage. ret->SetIsCroppingAreaOutSideOfSourceImage(data->GetSize(), aCropRect); return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateInternal(nsIGlobalObject* aGlobal, HTMLCanvasElement& aCanvasEl, const Maybe& aCropRect, ErrorResult& aRv) { if (aCanvasEl.Width() == 0 || aCanvasEl.Height() == 0) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } bool writeOnly = true; RefPtr surface = GetSurfaceFromElement(aGlobal, aCanvasEl, &writeOnly, aRv); if (NS_WARN_IF(aRv.Failed())) { return nullptr; } if (!writeOnly) { writeOnly = aCanvasEl.IsWriteOnly(); } // Crop the source surface if needed. RefPtr croppedSurface; IntRect cropRect = aCropRect.valueOr(IntRect()); // If the HTMLCanvasElement's rendering context is WebGL, then the snapshot // we got from the HTMLCanvasElement is a DataSourceSurface which is a copy // of the rendering context. We handle cropping in this case. if ((aCanvasEl.GetCurrentContextType() == CanvasContextType::WebGL1 || aCanvasEl.GetCurrentContextType() == CanvasContextType::WebGL2) && aCropRect.isSome()) { // The _surface_ must be a DataSourceSurface. MOZ_ASSERT(surface->GetType() == SurfaceType::DATA, "The snapshot SourceSurface from WebGL rendering contest is not DataSourceSurface."); RefPtr dataSurface = surface->GetDataSurface(); croppedSurface = CropAndCopyDataSourceSurface(dataSurface, cropRect); cropRect.MoveTo(0, 0); } else { croppedSurface = surface; } if (NS_WARN_IF(!croppedSurface)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } // Create an Image from the SourceSurface. RefPtr data = CreateImageFromSurface(croppedSurface); if (NS_WARN_IF(!data)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } RefPtr ret = new ImageBitmap(aGlobal, data, writeOnly); // Set the picture rectangle. if (ret && aCropRect.isSome()) { ret->SetPictureRect(cropRect, aRv); } // Set mIsCroppingAreaOutSideOfSourceImage. ret->SetIsCroppingAreaOutSideOfSourceImage(surface->GetSize(), aCropRect); return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateInternal(nsIGlobalObject* aGlobal, ImageData& aImageData, const Maybe& aCropRect, ErrorResult& aRv) { // Copy data into SourceSurface. dom::Uint8ClampedArray array; DebugOnly inited = array.Init(aImageData.GetDataObject()); MOZ_ASSERT(inited); array.ComputeLengthAndData(); const SurfaceFormat FORMAT = SurfaceFormat::R8G8B8A8; const uint32_t BYTES_PER_PIXEL = BytesPerPixel(FORMAT); const uint32_t imageWidth = aImageData.Width(); const uint32_t imageHeight = aImageData.Height(); const uint32_t imageStride = imageWidth * BYTES_PER_PIXEL; const uint32_t dataLength = array.Length(); const gfx::IntSize imageSize(imageWidth, imageHeight); // Check the ImageData is neutered or not. if (imageWidth == 0 || imageHeight == 0 || (imageWidth * imageHeight * BYTES_PER_PIXEL) != dataLength) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } // Create and Crop the raw data into a layers::Image RefPtr data; // The data could move during a GC; copy it out into a local buffer. uint8_t* fixedData = array.Data(); if (NS_IsMainThread()) { data = CreateImageFromRawData(imageSize, imageStride, FORMAT, fixedData, dataLength, aCropRect); } else { RefPtr task = new CreateImageFromRawDataInMainThreadSyncTask(fixedData, dataLength, imageStride, FORMAT, imageSize, aCropRect, getter_AddRefs(data)); task->Dispatch(Terminating, aRv); } if (NS_WARN_IF(!data)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } // Create an ImageBitmap. // ImageData's underlying data is not alpha-premultiplied. RefPtr ret = new ImageBitmap(aGlobal, data, false /* write-only */, false /* alpha-premult */); // The cropping information has been handled in the CreateImageFromRawData() // function. // Set mIsCroppingAreaOutSideOfSourceImage. ret->SetIsCroppingAreaOutSideOfSourceImage(imageSize, aCropRect); return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateInternal(nsIGlobalObject* aGlobal, CanvasRenderingContext2D& aCanvasCtx, const Maybe& aCropRect, ErrorResult& aRv) { // Check origin-clean bool writeOnly = aCanvasCtx.GetCanvas()->IsWriteOnly() || aCanvasCtx.IsWriteOnly(); RefPtr surface = aCanvasCtx.GetSurfaceSnapshot(); if (NS_WARN_IF(!surface)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } const IntSize surfaceSize = surface->GetSize(); if (surfaceSize.width == 0 || surfaceSize.height == 0) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } RefPtr data = CreateImageFromSurface(surface); if (NS_WARN_IF(!data)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } RefPtr ret = new ImageBitmap(aGlobal, data, writeOnly); // Set the picture rectangle. if (ret && aCropRect.isSome()) { ret->SetPictureRect(aCropRect.ref(), aRv); } // Set mIsCroppingAreaOutSideOfSourceImage. ret->SetIsCroppingAreaOutSideOfSourceImage(surface->GetSize(), aCropRect); return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateInternal(nsIGlobalObject* aGlobal, ImageBitmap& aImageBitmap, const Maybe& aCropRect, ErrorResult& aRv) { if (!aImageBitmap.mData) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } RefPtr data = aImageBitmap.mData; RefPtr ret = new ImageBitmap(aGlobal, data, aImageBitmap.mWriteOnly, aImageBitmap.mIsPremultipliedAlpha); // Set the picture rectangle. if (ret && aCropRect.isSome()) { ret->SetPictureRect(aCropRect.ref(), aRv); } // Set mIsCroppingAreaOutSideOfSourceImage. if (aImageBitmap.mIsCroppingAreaOutSideOfSourceImage == true) { ret->mIsCroppingAreaOutSideOfSourceImage = true; } else { ret->SetIsCroppingAreaOutSideOfSourceImage(aImageBitmap.mPictureRect.Size(), aCropRect); } return ret.forget(); } class FulfillImageBitmapPromise { protected: FulfillImageBitmapPromise(Promise* aPromise, ImageBitmap* aImageBitmap) : mPromise(aPromise) , mImageBitmap(aImageBitmap) { MOZ_ASSERT(aPromise); } void DoFulfillImageBitmapPromise() { mPromise->MaybeResolve(mImageBitmap); } private: RefPtr mPromise; RefPtr mImageBitmap; }; class FulfillImageBitmapPromiseTask final : public Runnable, public FulfillImageBitmapPromise { public: FulfillImageBitmapPromiseTask(Promise* aPromise, ImageBitmap* aImageBitmap) : FulfillImageBitmapPromise(aPromise, aImageBitmap) { } NS_IMETHOD Run() override { DoFulfillImageBitmapPromise(); return NS_OK; } }; class FulfillImageBitmapPromiseWorkerTask final : public WorkerSameThreadRunnable, public FulfillImageBitmapPromise { public: FulfillImageBitmapPromiseWorkerTask(Promise* aPromise, ImageBitmap* aImageBitmap) : WorkerSameThreadRunnable(GetCurrentThreadWorkerPrivate()), FulfillImageBitmapPromise(aPromise, aImageBitmap) { } bool WorkerRun(JSContext* aCx, WorkerPrivate* aWorkerPrivate) override { DoFulfillImageBitmapPromise(); return true; } }; static void AsyncFulfillImageBitmapPromise(Promise* aPromise, ImageBitmap* aImageBitmap) { if (NS_IsMainThread()) { nsCOMPtr task = new FulfillImageBitmapPromiseTask(aPromise, aImageBitmap); NS_DispatchToCurrentThread(task); // Actually, to the main-thread. } else { RefPtr task = new FulfillImageBitmapPromiseWorkerTask(aPromise, aImageBitmap); task->Dispatch(); // Actually, to the current worker-thread. } } static already_AddRefed DecodeBlob(Blob& aBlob) { // Get the internal stream of the blob. nsCOMPtr stream; ErrorResult error; aBlob.Impl()->GetInternalStream(getter_AddRefs(stream), error); if (NS_WARN_IF(error.Failed())) { error.SuppressException(); return nullptr; } // Get the MIME type string of the blob. // The type will be checked in the DecodeImage() method. nsAutoString mimeTypeUTF16; aBlob.GetType(mimeTypeUTF16); // Get the Component object. nsCOMPtr imgtool = do_GetService(NS_IMGTOOLS_CID); if (NS_WARN_IF(!imgtool)) { return nullptr; } // Decode image. NS_ConvertUTF16toUTF8 mimeTypeUTF8(mimeTypeUTF16); // NS_ConvertUTF16toUTF8 ---|> nsAutoCString nsCOMPtr imgContainer; nsresult rv = imgtool->DecodeImage(stream, mimeTypeUTF8, getter_AddRefs(imgContainer)); if (NS_WARN_IF(NS_FAILED(rv))) { return nullptr; } // Get the surface out. uint32_t frameFlags = imgIContainer::FLAG_SYNC_DECODE | imgIContainer::FLAG_WANT_DATA_SURFACE; uint32_t whichFrame = imgIContainer::FRAME_FIRST; RefPtr surface = imgContainer->GetFrame(whichFrame, frameFlags); if (NS_WARN_IF(!surface)) { return nullptr; } return surface.forget(); } static already_AddRefed DecodeAndCropBlob(Blob& aBlob, Maybe& aCropRect, /*Output*/ IntSize& sourceSize) { // Decode the blob into a SourceSurface. RefPtr surface = DecodeBlob(aBlob); if (NS_WARN_IF(!surface)) { return nullptr; } // Set the _sourceSize_ output parameter. sourceSize = surface->GetSize(); // Crop the source surface if needed. RefPtr croppedSurface = surface; if (aCropRect.isSome()) { // The blob is just decoded into a RasterImage and not optimized yet, so the // _surface_ we get is a DataSourceSurface which wraps the RasterImage's // raw buffer. // // The _surface_ might already be optimized so that its type is not // SurfaceType::DATA. However, we could keep using the generic cropping and // copying since the decoded buffer is only used in this ImageBitmap so we // should crop it to save memory usage. // // TODO: Bug1189632 is going to refactor this create-from-blob part to // decode the blob off the main thread. Re-check if we should do // cropping at this moment again there. RefPtr dataSurface = surface->GetDataSurface(); croppedSurface = CropAndCopyDataSourceSurface(dataSurface, aCropRect.ref()); aCropRect->MoveTo(0, 0); } if (NS_WARN_IF(!croppedSurface)) { return nullptr; } // Create an Image from the source surface. RefPtr image = CreateImageFromSurface(croppedSurface); if (NS_WARN_IF(!image)) { return nullptr; } return image.forget(); } class CreateImageBitmapFromBlob { protected: CreateImageBitmapFromBlob(Promise* aPromise, nsIGlobalObject* aGlobal, Blob& aBlob, const Maybe& aCropRect) : mPromise(aPromise), mGlobalObject(aGlobal), mBlob(&aBlob), mCropRect(aCropRect) { } virtual ~CreateImageBitmapFromBlob() { } // Returns true on success, false on failure. bool DoCreateImageBitmapFromBlob() { RefPtr imageBitmap = CreateImageBitmap(); // handle errors while creating ImageBitmap // (1) error occurs during reading of the object // (2) the image data is not in a supported file format // (3) the image data is corrupted // All these three cases should reject the promise with "InvalidStateError" // DOMException if (!imageBitmap) { return false; } if (imageBitmap && mCropRect.isSome()) { ErrorResult rv; imageBitmap->SetPictureRect(mCropRect.ref(), rv); if (rv.Failed()) { mPromise->MaybeReject(rv); return false; } } mPromise->MaybeResolve(imageBitmap); return true; } // Will return null on failure. In that case, mPromise will already // be rejected with the right thing. virtual already_AddRefed CreateImageBitmap() = 0; RefPtr mPromise; nsCOMPtr mGlobalObject; RefPtr mBlob; Maybe mCropRect; }; class CreateImageBitmapFromBlobTask final : public Runnable, public CreateImageBitmapFromBlob { public: CreateImageBitmapFromBlobTask(Promise* aPromise, nsIGlobalObject* aGlobal, Blob& aBlob, const Maybe& aCropRect) :CreateImageBitmapFromBlob(aPromise, aGlobal, aBlob, aCropRect) { } NS_IMETHOD Run() override { DoCreateImageBitmapFromBlob(); return NS_OK; } private: already_AddRefed CreateImageBitmap() override { // _sourceSize_ is used to get the original size of the source image, // before being cropped. IntSize sourceSize; // Keep the orignal cropping rectangle because the mCropRect might be // modified in DecodeAndCropBlob(). Maybe originalCropRect = mCropRect; RefPtr data = DecodeAndCropBlob(*mBlob, mCropRect, sourceSize); if (NS_WARN_IF(!data)) { mPromise->MaybeReject(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } // Create ImageBitmap object. RefPtr imageBitmap = new ImageBitmap(mGlobalObject, data, false /* write-only */); // Set mIsCroppingAreaOutSideOfSourceImage. imageBitmap->SetIsCroppingAreaOutSideOfSourceImage(sourceSize, originalCropRect); return imageBitmap.forget(); } }; class CreateImageBitmapFromBlobWorkerTask final : public WorkerSameThreadRunnable, public CreateImageBitmapFromBlob { // This is a synchronous task. class DecodeBlobInMainThreadSyncTask final : public WorkerMainThreadRunnable { public: DecodeBlobInMainThreadSyncTask(WorkerPrivate* aWorkerPrivate, Blob& aBlob, Maybe& aCropRect, layers::Image** aImage, IntSize& aSourceSize) : WorkerMainThreadRunnable(aWorkerPrivate, NS_LITERAL_CSTRING("ImageBitmap :: Create Image from Blob")) , mBlob(aBlob) , mCropRect(aCropRect) , mImage(aImage) , mSourceSize(aSourceSize) { } bool MainThreadRun() override { RefPtr image = DecodeAndCropBlob(mBlob, mCropRect, mSourceSize); if (NS_WARN_IF(!image)) { return true; } image.forget(mImage); return true; } private: Blob& mBlob; Maybe& mCropRect; layers::Image** mImage; IntSize mSourceSize; }; public: CreateImageBitmapFromBlobWorkerTask(Promise* aPromise, nsIGlobalObject* aGlobal, mozilla::dom::Blob& aBlob, const Maybe& aCropRect) : WorkerSameThreadRunnable(GetCurrentThreadWorkerPrivate()), CreateImageBitmapFromBlob(aPromise, aGlobal, aBlob, aCropRect) { } bool WorkerRun(JSContext* aCx, WorkerPrivate* aWorkerPrivate) override { return DoCreateImageBitmapFromBlob(); } private: already_AddRefed CreateImageBitmap() override { // _sourceSize_ is used to get the original size of the source image, // before being cropped. IntSize sourceSize; // Keep the orignal cropping rectangle because the mCropRect might be // modified in DecodeAndCropBlob(). Maybe originalCropRect = mCropRect; RefPtr data; ErrorResult rv; RefPtr task = new DecodeBlobInMainThreadSyncTask(mWorkerPrivate, *mBlob, mCropRect, getter_AddRefs(data), sourceSize); task->Dispatch(Terminating, rv); // This is a synchronous call. // In case the worker is terminating, this rejection can be handled. if (NS_WARN_IF(rv.Failed())) { mPromise->MaybeReject(rv); return nullptr; } if (NS_WARN_IF(!data)) { mPromise->MaybeReject(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } // Create ImageBitmap object. RefPtr imageBitmap = new ImageBitmap(mGlobalObject, data, false /* write-only */); // Set mIsCroppingAreaOutSideOfSourceImage. imageBitmap->SetIsCroppingAreaOutSideOfSourceImage(sourceSize, originalCropRect); return imageBitmap.forget(); } }; static void AsyncCreateImageBitmapFromBlob(Promise* aPromise, nsIGlobalObject* aGlobal, Blob& aBlob, const Maybe& aCropRect) { if (NS_IsMainThread()) { nsCOMPtr task = new CreateImageBitmapFromBlobTask(aPromise, aGlobal, aBlob, aCropRect); NS_DispatchToCurrentThread(task); // Actually, to the main-thread. } else { RefPtr task = new CreateImageBitmapFromBlobWorkerTask(aPromise, aGlobal, aBlob, aCropRect); task->Dispatch(); // Actually, to the current worker-thread. } } /* static */ already_AddRefed ImageBitmap::Create(nsIGlobalObject* aGlobal, const ImageBitmapSource& aSrc, const Maybe& aCropRect, ErrorResult& aRv) { MOZ_ASSERT(aGlobal); RefPtr promise = Promise::Create(aGlobal, aRv); if (NS_WARN_IF(aRv.Failed())) { return nullptr; } if (aCropRect.isSome() && (aCropRect->Width() == 0 || aCropRect->Height() == 0)) { aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); return promise.forget(); } RefPtr imageBitmap; if (aSrc.IsHTMLImageElement()) { MOZ_ASSERT(NS_IsMainThread(), "Creating ImageBitmap from HTMLImageElement off the main thread."); imageBitmap = CreateInternal(aGlobal, aSrc.GetAsHTMLImageElement(), aCropRect, aRv); } else if (aSrc.IsHTMLVideoElement()) { MOZ_ASSERT(NS_IsMainThread(), "Creating ImageBitmap from HTMLVideoElement off the main thread."); imageBitmap = CreateInternal(aGlobal, aSrc.GetAsHTMLVideoElement(), aCropRect, aRv); } else if (aSrc.IsHTMLCanvasElement()) { MOZ_ASSERT(NS_IsMainThread(), "Creating ImageBitmap from HTMLCanvasElement off the main thread."); imageBitmap = CreateInternal(aGlobal, aSrc.GetAsHTMLCanvasElement(), aCropRect, aRv); } else if (aSrc.IsImageData()) { imageBitmap = CreateInternal(aGlobal, aSrc.GetAsImageData(), aCropRect, aRv); } else if (aSrc.IsCanvasRenderingContext2D()) { MOZ_ASSERT(NS_IsMainThread(), "Creating ImageBitmap from CanvasRenderingContext2D off the main thread."); imageBitmap = CreateInternal(aGlobal, aSrc.GetAsCanvasRenderingContext2D(), aCropRect, aRv); } else if (aSrc.IsImageBitmap()) { imageBitmap = CreateInternal(aGlobal, aSrc.GetAsImageBitmap(), aCropRect, aRv); } else if (aSrc.IsBlob()) { AsyncCreateImageBitmapFromBlob(promise, aGlobal, aSrc.GetAsBlob(), aCropRect); return promise.forget(); } else { aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); } if (!aRv.Failed()) { AsyncFulfillImageBitmapPromise(promise, imageBitmap); } return promise.forget(); } /*static*/ JSObject* ImageBitmap::ReadStructuredClone(JSContext* aCx, JSStructuredCloneReader* aReader, nsIGlobalObject* aParent, const nsTArray>& aClonedSurfaces, uint32_t aIndex) { MOZ_ASSERT(aCx); MOZ_ASSERT(aReader); // aParent might be null. uint32_t picRectX_; uint32_t picRectY_; uint32_t picRectWidth_; uint32_t picRectHeight_; uint32_t isPremultipliedAlpha_; uint32_t isCroppingAreaOutSideOfSourceImage_; uint32_t writeOnly; uint32_t dummy; if (!JS_ReadUint32Pair(aReader, &picRectX_, &picRectY_) || !JS_ReadUint32Pair(aReader, &picRectWidth_, &picRectHeight_) || !JS_ReadUint32Pair(aReader, &isPremultipliedAlpha_, &isCroppingAreaOutSideOfSourceImage_) || !JS_ReadUint32Pair(aReader, &writeOnly, &dummy)) { return nullptr; } MOZ_ASSERT(dummy == 0); int32_t picRectX = BitwiseCast(picRectX_); int32_t picRectY = BitwiseCast(picRectY_); int32_t picRectWidth = BitwiseCast(picRectWidth_); int32_t picRectHeight = BitwiseCast(picRectHeight_); // Create a new ImageBitmap. MOZ_ASSERT(!aClonedSurfaces.IsEmpty()); MOZ_ASSERT(aIndex < aClonedSurfaces.Length()); // RefPtr needs to go out of scope before toObjectOrNull() is // called because the static analysis thinks dereferencing XPCOM objects // can GC (because in some cases it can!), and a return statement with a // JSObject* type means that JSObject* is on the stack as a raw pointer // while destructors are running. JS::Rooted value(aCx); { RefPtr img = CreateImageFromSurface(aClonedSurfaces[aIndex]); RefPtr imageBitmap = new ImageBitmap(aParent, img, !!writeOnly, isPremultipliedAlpha_); imageBitmap->mIsCroppingAreaOutSideOfSourceImage = isCroppingAreaOutSideOfSourceImage_; ErrorResult error; imageBitmap->SetPictureRect(IntRect(picRectX, picRectY, picRectWidth, picRectHeight), error); if (NS_WARN_IF(error.Failed())) { error.SuppressException(); return nullptr; } if (!GetOrCreateDOMReflector(aCx, imageBitmap, &value)) { return nullptr; } } return &(value.toObject()); } /*static*/ bool ImageBitmap::WriteStructuredClone(JSStructuredCloneWriter* aWriter, nsTArray>& aClonedSurfaces, ImageBitmap* aImageBitmap) { MOZ_ASSERT(aWriter); MOZ_ASSERT(aImageBitmap); const uint32_t picRectX = BitwiseCast(aImageBitmap->mPictureRect.x); const uint32_t picRectY = BitwiseCast(aImageBitmap->mPictureRect.y); const uint32_t picRectWidth = BitwiseCast(aImageBitmap->mPictureRect.width); const uint32_t picRectHeight = BitwiseCast(aImageBitmap->mPictureRect.height); const uint32_t isPremultipliedAlpha = aImageBitmap->mIsPremultipliedAlpha ? 1 : 0; const uint32_t isCroppingAreaOutSideOfSourceImage = aImageBitmap->mIsCroppingAreaOutSideOfSourceImage ? 1 : 0; const uint32_t isWriteOnly = aImageBitmap->mWriteOnly ? 1 : 0; // Indexing the cloned surfaces and send the index to the receiver. uint32_t index = aClonedSurfaces.Length(); if (NS_WARN_IF(!JS_WriteUint32Pair(aWriter, SCTAG_DOM_IMAGEBITMAP, index)) || NS_WARN_IF(!JS_WriteUint32Pair(aWriter, picRectX, picRectY)) || NS_WARN_IF(!JS_WriteUint32Pair(aWriter, picRectWidth, picRectHeight)) || NS_WARN_IF(!JS_WriteUint32Pair(aWriter, isPremultipliedAlpha, isCroppingAreaOutSideOfSourceImage)) || NS_WARN_IF(!JS_WriteUint32Pair(aWriter, isWriteOnly, 0))) { return false; } RefPtr surface = aImageBitmap->mData->GetAsSourceSurface(); RefPtr snapshot = surface->GetDataSurface(); RefPtr dstDataSurface; { // DataSourceSurfaceD2D1::GetStride() will call EnsureMapped implicitly and // won't Unmap after exiting function. So instead calling GetStride() // directly, using ScopedMap to get stride. DataSourceSurface::ScopedMap map(snapshot, DataSourceSurface::READ); dstDataSurface = Factory::CreateDataSourceSurfaceWithStride(snapshot->GetSize(), snapshot->GetFormat(), map.GetStride(), true); } MOZ_ASSERT(dstDataSurface); Factory::CopyDataSourceSurface(snapshot, dstDataSurface); aClonedSurfaces.AppendElement(dstDataSurface); return true; } /*static*/ bool ImageBitmap::ExtensionsEnabled(JSContext* aCx, JSObject*) { if (NS_IsMainThread()) { return Preferences::GetBool("canvas.imagebitmap_extensions.enabled"); } else { WorkerPrivate* workerPrivate = GetWorkerPrivateFromContext(aCx); MOZ_ASSERT(workerPrivate); return workerPrivate->ImageBitmapExtensionsEnabled(); } } // ImageBitmap extensions. ImageBitmapFormat ImageBitmap::FindOptimalFormat(const Optional>& aPossibleFormats, ErrorResult& aRv) { MOZ_ASSERT(mDataWrapper, "No ImageBitmapFormatUtils functionalities."); ImageBitmapFormat platformFormat = mDataWrapper->GetFormat(); if (!aPossibleFormats.WasPassed() || aPossibleFormats.Value().Contains(platformFormat)) { return platformFormat; } else { // If no matching is found, FindBestMatchingFromat() returns // ImageBitmapFormat::EndGuard_ and we throw an exception. ImageBitmapFormat optimalFormat = FindBestMatchingFromat(platformFormat, aPossibleFormats.Value()); if (optimalFormat == ImageBitmapFormat::EndGuard_) { aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); } return optimalFormat; } } int32_t ImageBitmap::MappedDataLength(ImageBitmapFormat aFormat, ErrorResult& aRv) { MOZ_ASSERT(mDataWrapper, "No ImageBitmapFormatUtils functionalities."); if (aFormat == mDataWrapper->GetFormat()) { return mDataWrapper->GetBufferLength(); } else { return CalculateImageBufferSize(aFormat, Width(), Height()); } } template class MapDataIntoBufferSource { protected: MapDataIntoBufferSource(JSContext* aCx, Promise *aPromise, ImageBitmap *aImageBitmap, const T& aBuffer, int32_t aOffset, ImageBitmapFormat aFormat) : mPromise(aPromise) , mImageBitmap(aImageBitmap) , mBuffer(aCx, aBuffer.Obj()) , mOffset(aOffset) , mFormat(aFormat) { MOZ_ASSERT(mPromise); MOZ_ASSERT(JS_IsArrayBufferObject(mBuffer) || JS_IsArrayBufferViewObject(mBuffer)); } virtual ~MapDataIntoBufferSource() = default; void DoMapDataIntoBufferSource() { ErrorResult error; // Prepare destination buffer. uint8_t* bufferData = nullptr; uint32_t bufferLength = 0; bool isSharedMemory = false; if (JS_IsArrayBufferObject(mBuffer)) { js::GetArrayBufferLengthAndData(mBuffer, &bufferLength, &isSharedMemory, &bufferData); } else if (JS_IsArrayBufferViewObject(mBuffer)) { js::GetArrayBufferViewLengthAndData(mBuffer, &bufferLength, &isSharedMemory, &bufferData); } else { error.Throw(NS_ERROR_NOT_IMPLEMENTED); mPromise->MaybeReject(error); return; } if (NS_WARN_IF(!bufferData) || NS_WARN_IF(!bufferLength)) { error.Throw(NS_ERROR_NOT_AVAILABLE); mPromise->MaybeReject(error); return; } // Check length. const int32_t neededBufferLength = mImageBitmap->MappedDataLength(mFormat, error); if (((int32_t)bufferLength - mOffset) < neededBufferLength) { error.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); mPromise->MaybeReject(error); return; } // Call ImageBitmapFormatUtils. UniquePtr layout = mImageBitmap->mDataWrapper->MapDataInto(bufferData, mOffset, bufferLength, mFormat, error); if (NS_WARN_IF(!layout)) { mPromise->MaybeReject(error); return; } mPromise->MaybeResolve(*layout); } RefPtr mPromise; RefPtr mImageBitmap; JS::PersistentRooted mBuffer; int32_t mOffset; ImageBitmapFormat mFormat; }; template class MapDataIntoBufferSourceTask final : public Runnable, public MapDataIntoBufferSource { public: MapDataIntoBufferSourceTask(JSContext* aCx, Promise *aPromise, ImageBitmap *aImageBitmap, const T& aBuffer, int32_t aOffset, ImageBitmapFormat aFormat) : MapDataIntoBufferSource(aCx, aPromise, aImageBitmap, aBuffer, aOffset, aFormat) { } virtual ~MapDataIntoBufferSourceTask() = default; NS_IMETHOD Run() override { MapDataIntoBufferSource::DoMapDataIntoBufferSource(); return NS_OK; } }; template class MapDataIntoBufferSourceWorkerTask final : public WorkerSameThreadRunnable, public MapDataIntoBufferSource { public: MapDataIntoBufferSourceWorkerTask(JSContext* aCx, Promise *aPromise, ImageBitmap *aImageBitmap, const T& aBuffer, int32_t aOffset, ImageBitmapFormat aFormat) : WorkerSameThreadRunnable(GetCurrentThreadWorkerPrivate()), MapDataIntoBufferSource(aCx, aPromise, aImageBitmap, aBuffer, aOffset, aFormat) { } virtual ~MapDataIntoBufferSourceWorkerTask() = default; bool WorkerRun(JSContext* aCx, WorkerPrivate* aWorkerPrivate) override { MapDataIntoBufferSource::DoMapDataIntoBufferSource(); return true; } }; void AsyncMapDataIntoBufferSource(JSContext* aCx, Promise *aPromise, ImageBitmap *aImageBitmap, const ArrayBufferViewOrArrayBuffer& aBuffer, int32_t aOffset, ImageBitmapFormat aFormat) { MOZ_ASSERT(aCx); MOZ_ASSERT(aPromise); MOZ_ASSERT(aImageBitmap); if (NS_IsMainThread()) { nsCOMPtr task; if (aBuffer.IsArrayBuffer()) { const ArrayBuffer& buffer = aBuffer.GetAsArrayBuffer(); task = new MapDataIntoBufferSourceTask(aCx, aPromise, aImageBitmap, buffer, aOffset, aFormat); } else if (aBuffer.IsArrayBufferView()) { const ArrayBufferView& bufferView = aBuffer.GetAsArrayBufferView(); task = new MapDataIntoBufferSourceTask(aCx, aPromise, aImageBitmap, bufferView, aOffset, aFormat); } NS_DispatchToCurrentThread(task); // Actually, to the main-thread. } else { RefPtr task; if (aBuffer.IsArrayBuffer()) { const ArrayBuffer& buffer = aBuffer.GetAsArrayBuffer(); task = new MapDataIntoBufferSourceWorkerTask(aCx, aPromise, aImageBitmap, buffer, aOffset, aFormat); } else if (aBuffer.IsArrayBufferView()) { const ArrayBufferView& bufferView = aBuffer.GetAsArrayBufferView(); task = new MapDataIntoBufferSourceWorkerTask(aCx, aPromise, aImageBitmap, bufferView, aOffset, aFormat); } task->Dispatch(); // Actually, to the current worker-thread. } } already_AddRefed ImageBitmap::MapDataInto(JSContext* aCx, ImageBitmapFormat aFormat, const ArrayBufferViewOrArrayBuffer& aBuffer, int32_t aOffset, ErrorResult& aRv) { MOZ_ASSERT(mDataWrapper, "No ImageBitmapFormatUtils functionalities."); MOZ_ASSERT(aCx, "No JSContext while calling ImageBitmap::MapDataInto()."); RefPtr promise = Promise::Create(mParent, aRv); if (NS_WARN_IF(aRv.Failed())) { return nullptr; } // Check for cases that should throws. // Case 1: // If image bitmap was cropped to the source rectangle so that it contains any // transparent black pixels (cropping area is outside of the source image), // then reject promise with IndexSizeError and abort these steps. if (mIsCroppingAreaOutSideOfSourceImage) { aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); return promise.forget(); } // Case 2: // If the image bitmap is going to be accessed in YUV422/YUV422 series with a // cropping area starts at an odd x or y coordinate. if (aFormat == ImageBitmapFormat::YUV422P || aFormat == ImageBitmapFormat::YUV420P || aFormat == ImageBitmapFormat::YUV420SP_NV12 || aFormat == ImageBitmapFormat::YUV420SP_NV21) { if ((mPictureRect.x & 1) || (mPictureRect.y & 1)) { aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); return promise.forget(); } } AsyncMapDataIntoBufferSource(aCx, promise, this, aBuffer, aOffset, aFormat); return promise.forget(); } // ImageBitmapFactories extensions. static SurfaceFormat ImageFormatToSurfaceFromat(mozilla::dom::ImageBitmapFormat aFormat) { switch(aFormat) { case ImageBitmapFormat::RGBA32: return SurfaceFormat::R8G8B8A8; case ImageBitmapFormat::BGRA32: return SurfaceFormat::B8G8R8A8; case ImageBitmapFormat::RGB24: return SurfaceFormat::R8G8B8; case ImageBitmapFormat::BGR24: return SurfaceFormat::B8G8R8; case ImageBitmapFormat::GRAY8: return SurfaceFormat::A8; case ImageBitmapFormat::HSV: return SurfaceFormat::HSV; case ImageBitmapFormat::Lab: return SurfaceFormat::Lab; case ImageBitmapFormat::DEPTH: return SurfaceFormat::Depth; default: return SurfaceFormat::UNKNOWN; } } static already_AddRefed CreateImageFromBufferSourceRawData(const uint8_t*aBufferData, uint32_t aBufferLength, mozilla::dom::ImageBitmapFormat aFormat, const Sequence& aLayout) { MOZ_ASSERT(aBufferData); MOZ_ASSERT(aBufferLength > 0); switch(aFormat) { case ImageBitmapFormat::RGBA32: case ImageBitmapFormat::BGRA32: case ImageBitmapFormat::RGB24: case ImageBitmapFormat::BGR24: case ImageBitmapFormat::GRAY8: case ImageBitmapFormat::HSV: case ImageBitmapFormat::Lab: case ImageBitmapFormat::DEPTH: { const nsTArray& channels = aLayout; MOZ_ASSERT(channels.Length() != 0, "Empty Channels."); const SurfaceFormat srcFormat = ImageFormatToSurfaceFromat(aFormat); const uint32_t srcStride = channels[0].mStride; const IntSize srcSize(channels[0].mWidth, channels[0].mHeight); RefPtr dstDataSurface = Factory::CreateDataSourceSurfaceWithStride(srcSize, srcFormat, srcStride); if (NS_WARN_IF(!dstDataSurface)) { return nullptr; } // Copy the raw data into the newly created DataSourceSurface. DataSourceSurface::ScopedMap dstMap(dstDataSurface, DataSourceSurface::WRITE); if (NS_WARN_IF(!dstMap.IsMapped())) { return nullptr; } const uint8_t* srcBufferPtr = aBufferData; uint8_t* dstBufferPtr = dstMap.GetData(); for (int i = 0; i < srcSize.height; ++i) { memcpy(dstBufferPtr, srcBufferPtr, srcStride); srcBufferPtr += srcStride; dstBufferPtr += dstMap.GetStride(); } // Create an Image from the BGRA SourceSurface. RefPtr surface = dstDataSurface; RefPtr image = CreateImageFromSurface(surface); if (NS_WARN_IF(!image)) { return nullptr; } return image.forget(); } case ImageBitmapFormat::YUV444P: case ImageBitmapFormat::YUV422P: case ImageBitmapFormat::YUV420P: case ImageBitmapFormat::YUV420SP_NV12: case ImageBitmapFormat::YUV420SP_NV21: { // Prepare the PlanarYCbCrData. const ChannelPixelLayout& yLayout = aLayout[0]; const ChannelPixelLayout& uLayout = aFormat != ImageBitmapFormat::YUV420SP_NV21 ? aLayout[1] : aLayout[2]; const ChannelPixelLayout& vLayout = aFormat != ImageBitmapFormat::YUV420SP_NV21 ? aLayout[2] : aLayout[1]; layers::PlanarYCbCrData data; // Luminance buffer data.mYChannel = const_cast(aBufferData + yLayout.mOffset); data.mYStride = yLayout.mStride; data.mYSize = gfx::IntSize(yLayout.mWidth, yLayout.mHeight); data.mYSkip = yLayout.mSkip; // Chroma buffers data.mCbChannel = const_cast(aBufferData + uLayout.mOffset); data.mCrChannel = const_cast(aBufferData + vLayout.mOffset); data.mCbCrStride = uLayout.mStride; data.mCbCrSize = gfx::IntSize(uLayout.mWidth, uLayout.mHeight); data.mCbSkip = uLayout.mSkip; data.mCrSkip = vLayout.mSkip; // Picture rectangle. // We set the picture rectangle to exactly the size of the source image to // keep the full original data. data.mPicX = 0; data.mPicY = 0; data.mPicSize = data.mYSize; // Create a layers::Image and set data. if (aFormat == ImageBitmapFormat::YUV444P || aFormat == ImageBitmapFormat::YUV422P || aFormat == ImageBitmapFormat::YUV420P) { RefPtr image = new layers::RecyclingPlanarYCbCrImage(new layers::BufferRecycleBin()); if (NS_WARN_IF(!image)) { return nullptr; } // Set Data. if (NS_WARN_IF(!image->CopyData(data))) { return nullptr; } return image.forget(); } else { RefPtrimage = new layers::NVImage(); if (NS_WARN_IF(!image)) { return nullptr; } // Set Data. if (NS_WARN_IF(!image->SetData(data))) { return nullptr; } return image.forget(); } } default: return nullptr; } } /* * This is a synchronous task. * This class is used to create a layers::CairoImage from raw data in the main * thread. While creating an ImageBitmap from an BufferSource, we need to create * a SouceSurface from the BufferSource raw data and then set the SourceSurface * into a layers::CairoImage. However, the layers::CairoImage asserts the * setting operation in the main thread, so if we are going to create an * ImageBitmap from an BufferSource off the main thread, we post an event to the * main thread to create a layers::CairoImage from an BufferSource raw data. * * TODO: Once the layers::CairoImage is constructible off the main thread, which * means the SouceSurface could be released anywhere, we do not need this * task anymore. */ class CreateImageFromBufferSourceRawDataInMainThreadSyncTask final : public WorkerMainThreadRunnable { public: CreateImageFromBufferSourceRawDataInMainThreadSyncTask(const uint8_t* aBuffer, uint32_t aBufferLength, mozilla::dom::ImageBitmapFormat aFormat, const Sequence& aLayout, /*output*/ layers::Image** aImage) : WorkerMainThreadRunnable(GetCurrentThreadWorkerPrivate(), NS_LITERAL_CSTRING("ImageBitmap-extensions :: Create Image from BufferSource Raw Data")) , mImage(aImage) , mBuffer(aBuffer) , mBufferLength(aBufferLength) , mFormat(aFormat) , mLayout(aLayout) { MOZ_ASSERT(!(*aImage), "Don't pass an existing Image into CreateImageFromBufferSourceRawDataInMainThreadSyncTask."); } bool MainThreadRun() override { RefPtr image = CreateImageFromBufferSourceRawData(mBuffer, mBufferLength, mFormat, mLayout); if (NS_WARN_IF(!image)) { return true; } image.forget(mImage); return true; } private: layers::Image** mImage; const uint8_t* mBuffer; uint32_t mBufferLength; mozilla::dom::ImageBitmapFormat mFormat; const Sequence& mLayout; }; /*static*/ already_AddRefed ImageBitmap::Create(nsIGlobalObject* aGlobal, const ImageBitmapSource& aBuffer, int32_t aOffset, int32_t aLength, mozilla::dom::ImageBitmapFormat aFormat, const Sequence& aLayout, ErrorResult& aRv) { MOZ_ASSERT(aGlobal); RefPtr promise = Promise::Create(aGlobal, aRv); if (NS_WARN_IF(aRv.Failed())) { return nullptr; } uint8_t* bufferData = nullptr; uint32_t bufferLength = 0; if (aBuffer.IsArrayBuffer()) { const ArrayBuffer& buffer = aBuffer.GetAsArrayBuffer(); buffer.ComputeLengthAndData(); bufferData = buffer.Data(); bufferLength = buffer.Length(); } else if (aBuffer.IsArrayBufferView()) { const ArrayBufferView& bufferView = aBuffer.GetAsArrayBufferView(); bufferView.ComputeLengthAndData(); bufferData = bufferView.Data(); bufferLength = bufferView.Length(); } else { aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); return promise.forget(); } MOZ_ASSERT(bufferData && bufferLength > 0, "Cannot read data from BufferSource."); // Check the buffer. if (((uint32_t)(aOffset + aLength) > bufferLength)) { aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); return promise.forget(); } // Create and Crop the raw data into a layers::Image RefPtr data; if (NS_IsMainThread()) { data = CreateImageFromBufferSourceRawData(bufferData + aOffset, bufferLength, aFormat, aLayout); } else { RefPtr task = new CreateImageFromBufferSourceRawDataInMainThreadSyncTask(bufferData + aOffset, bufferLength, aFormat, aLayout, getter_AddRefs(data)); task->Dispatch(Terminating, aRv); if (aRv.Failed()) { return promise.forget(); } } if (NS_WARN_IF(!data)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return promise.forget(); } // Create an ImageBimtap. // Assume the data from an external buffer is not alpha-premultiplied. RefPtr imageBitmap = new ImageBitmap(aGlobal, data, false); // We don't need to call SetPictureRect() here because there is no cropping // supported and the ImageBitmap's mPictureRect is the size of the source // image in default // We don't need to set mIsCroppingAreaOutSideOfSourceImage here because there // is no cropping supported and the mIsCroppingAreaOutSideOfSourceImage is // false in default. AsyncFulfillImageBitmapPromise(promise, imageBitmap); return promise.forget(); } } // namespace dom } // namespace mozilla