U 4Af@s UdZddlZddlZddlmZddlmZmZmZm Z m Z m Z m Z ddl ZddlmZmZmZddlmZmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!dd l"m#Z#m$Z$m%Z%m&Z&e%rddl'Z'e&(e)Z*eZ+eZ,d d d gZ-d d d gZ.d Z/e0e1d <GdddZ2GdddeZ3dS)z Image processor class for Flava.N) lru_cache)AnyDictIterableListOptionalTupleUnion)BaseImageProcessor BatchFeature get_size_dict)resizeto_channel_dimension_format) OPENAI_CLIP_MEANOPENAI_CLIP_STDChannelDimension ImageInputPILImageResamplinginfer_channel_dimension_formatis_scaled_imagemake_list_of_imagesto_numpy_array valid_imagesvalidate_preprocess_arguments) TensorTypefilter_out_non_signature_kwargsis_vision_availablelogginggg?g?LOGIT_LAPLACE_EPSc@s^eZdZdeeeeeffeeeeeeedddZd d Z d d Z d dZ ddZ dS)FlavaMaskingGeneratorKN333333?) input_sizetotal_mask_patchesmask_group_max_patchesmask_group_min_patchesmask_group_min_aspect_ratiomask_group_max_aspect_ratiocCsrt|ts|fd}|\|_|_|j|j|_||_||_|dkrF|n||_|pVd|}t |t |f|_ dS)N) isinstancetupleheightwidthZ num_patchesr&r(r'mathloglog_aspect_ratio)selfr%r&r'r(r)r*r5T/tmp/pip-unpacked-wheel-zw5xktn0/transformers/models/flava/image_processing_flava.py__init__;s    zFlavaMaskingGenerator.__init__c Cs0d|j|j|j|j|j|jd|jdf}|S)Nz [%d ~ %d], max = %d, %.3f ~ %.3f)rr,)r/r0r(r'r&r3)r4repr_strr5r5r6__repr__Qs zFlavaMaskingGenerator.__repr__cCs |j|jfS)Nr/r0)r4r5r5r6 get_shape]szFlavaMaskingGenerator.get_shapecCs8d}tdD]$}t|j|}ttj|j}ttt ||}ttt ||}||j kr ||j kr t d|j |} t d|j |} || | || | |f } d||| kr|kr$nnLt| | |D]<} t| | |D](} || | fdkrd|| | f<|d7}qq|dkr q4q |S)Nr r,)rangerandomuniformr(r1expr3introundsqrtr0r/randintsum)r4maskmax_mask_patchesdeltaZ_attemptZ target_areaZ aspect_ratior/r0topleftZ num_maskedijr5r5r6_mask`s& "  zFlavaMaskingGenerator._maskcCs\tj|td}d}||jkrX|j|}t||j}|||}|dkrNqXq||7}q|S)N)shapeZdtyper)npzerosr;rAr&minr'rM)r4rFZ mask_countrGrHr5r5r6__call__xs     zFlavaMaskingGenerator.__call__)r!r"Nr#r$N) __name__ __module__ __qualname__r rArrfloatr7r9r;rMrRr5r5r5r6r :s$  r c&s~eZdZdZdgZddejddddddddddd dd ddddejddddddddfee e e feee e e fee e e feee e ee fee e ee fee e e ee e ee eeee ee ee e e feeee e ee fee e ee fdd fd d Zee e efdfdd ZeedddZejddfeje e e feee e efee e efejdddZejejdddZdddddddddddejdf eee e e feee e e fee eee e ee fee e ee feeeeeejdddZe ddddddddddddddddddddddddddddddejdf eeee e e feeeee e e feeee eeee e ee fee e ee feeee ee ee ee ee ee eeeeee e e fee eeee e e feeee eeeeeee eee ee e e!feee e efe"j#j#d"ddZ$Z%S) FlavaImageProcessora  Constructs a Flava image processor. Args: do_resize (`bool`, *optional*, defaults to `True`): Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the `do_resize` parameter in `preprocess`. size (`Dict[str, int]` *optional*, defaults to `{"height": 224, "width": 224}`): Size of the image after resizing. Can be overridden by the `size` parameter in `preprocess`. resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): Resampling filter to use if resizing the image. Can be overridden by the `resample` parameter in `preprocess`. do_center_crop (`bool`, *optional*, defaults to `True`): Whether to center crop the images. Can be overridden by the `do_center_crop` parameter in `preprocess`. crop_size (`Dict[str, int]` *optional*, defaults to `{"height": 224, "width": 224}`): Size of image after the center crop `(crop_size["height"], crop_size["width"])`. Can be overridden by the `crop_size` parameter in `preprocess`. do_rescale (`bool`, *optional*, defaults to `True`): Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the `do_rescale` parameter in `preprocess`. rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): Scale factor to use if rescaling the image. Can be overridden by the `rescale_factor` parameter in `preprocess`. do_normalize (`bool`, *optional*, defaults to `True`): Whether to normalize the image. Can be overridden by the `do_normalize` parameter in `preprocess`. image_mean (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_MEAN`): Mean to use if normalizing the image. This is a float or list of floats the length of the number of channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. image_std (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_STD`): Standard deviation to use if normalizing the image. This is a float or list of floats the length of the number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. return_image_mask (`bool`, *optional*, defaults to `False`): Whether to return the image mask. Can be overridden by the `return_image_mask` parameter in `preprocess`. input_size_patches (`int`, *optional*, defaults to 14): Number of patches in the image in height and width direction. 14x14 = 196 total patches. Can be overridden by the `input_size_patches` parameter in `preprocess`. total_mask_patches (`int`, *optional*, defaults to 75): Total number of patches that should be masked. Can be overridden by the `total_mask_patches` parameter in `preprocess`. mask_group_min_patches (`int`, *optional*, defaults to 16): Minimum number of patches that should be masked. Can be overridden by the `mask_group_min_patches` parameter in `preprocess`. mask_group_max_patches (`int`, *optional*): Maximum number of patches that should be masked. Can be overridden by the `mask_group_max_patches` parameter in `preprocess`. mask_group_min_aspect_ratio (`float`, *optional*, defaults to 0.3): Minimum aspect ratio of the mask window. Can be overridden by the `mask_group_min_aspect_ratio` parameter in `preprocess`. mask_group_max_aspect_ratio (`float`, *optional*): Maximum aspect ratio of the mask window. Can be overridden by the `mask_group_max_aspect_ratio` parameter in `preprocess`. codebook_do_resize (`bool`, *optional*, defaults to `True`): Whether to resize the input for codebook to a certain. Can be overridden by the `codebook_do_resize` parameter in `preprocess`. `codebook_size`. codebook_size (`Dict[str, int]`, *optional*, defaults to `{"height": 224, "width": 224}`): Resize the input for codebook to the given size. Can be overridden by the `codebook_size` parameter in `preprocess`. codebook_resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.LANCZOS`): Resampling filter to use if resizing the codebook image. Can be overridden by the `codebook_resample` parameter in `preprocess`. codebook_do_center_crop (`bool`, *optional*, defaults to `True`): Whether to crop the input for codebook at the center. If the input size is smaller than `codebook_crop_size` along any edge, the image is padded with 0's and then center cropped. Can be overridden by the `codebook_do_center_crop` parameter in `preprocess`. codebook_crop_size (`Dict[str, int]`, *optional*, defaults to `{"height": 224, "width": 224}`): Desired output size for codebook input when applying center-cropping. Can be overridden by the `codebook_crop_size` parameter in `preprocess`. codebook_do_rescale (`bool`, *optional*, defaults to `True`): Whether to rescale the input for codebook by the specified scale `codebook_rescale_factor`. Can be overridden by the `codebook_do_rescale` parameter in `preprocess`. codebook_rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): Defines the scale factor to use if rescaling the codebook image. Can be overridden by the `codebook_rescale_factor` parameter in `preprocess`. codebook_do_map_pixels (`bool`, *optional*, defaults to `True`): Whether to map the pixel values of the codebook input to (1 - 2e)x + e. Can be overridden by the `codebook_do_map_pixels` parameter in `preprocess`. codebook_do_normalize (`bool`, *optional*, defaults to `True`): Whether or not to normalize the input for codebook with `codebook_image_mean` and `codebook_image_std`. Can be overridden by the `codebook_do_normalize` parameter in `preprocess`. codebook_image_mean (`Optional[Union[float, Iterable[float]]]`, *optional*, defaults to `[0, 0, 0]`): The sequence of means for each channel, to be used when normalizing images for codebook. Can be overridden by the `codebook_image_mean` parameter in `preprocess`. codebook_image_std (`Optional[Union[float, Iterable[float]]]`, *optional*, defaults to `[0.5, 0.5, 0.5]`): The sequence of standard deviations for each channel, to be used when normalizing images for codebook. Can be overridden by the `codebook_image_std` parameter in `preprocess`. pixel_valuesTNgp?Fr!r"r#r$) do_resizesizeresampledo_center_crop crop_size do_rescalerescale_factor do_normalize image_mean image_stdreturn_image_maskinput_size_patchesr&r(r'r)r*return_codebook_pixelscodebook_do_resize codebook_sizecodebook_resamplecodebook_do_center_cropcodebook_crop_sizecodebook_do_rescalecodebook_rescale_factorcodebook_do_map_pixelscodebook_do_normalizecodebook_image_meancodebook_image_stdreturnc s~tjf||dk r|nddd}t|}|dk r8|nddd}t|dd}|dk rZ|nddd}t|dd}|dk r||nddd}t|dd}||_||_||_||_||_||_||_ ||_ | dk r| nt |_ | dk r| nt |_| |_| |_| |_||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_ |dk r`|nt!|_ |dk rt|nt"|_#dS)Nr:r] param_nameprgrj)$superr7r rYrZr[r^r_r\r]r`FLAVA_IMAGE_MEANraFLAVA_IMAGE_STDrbrcrdr&r(r'r)r*rerfrgrhrirjrkrlrmrnroFLAVA_CODEBOOK_MEANFLAVA_CODEBOOK_STDrp)r4rYrZr[r\r]r^r_r`rarbrcrdr&r(r'r)r*rerfrgrhrirjrkrlrmrnrorpkwargs __class__r5r6r7sN#   zFlavaImageProcessor.__init__)image_processor_dictc sD|}d|kr|d|d<d|kr4|d|d<tj|f|S)z Overrides the `from_dict` method from the base class to make sure parameters are updated if image processor is created using from_dict and kwargs e.g. `FlavaImageProcessor.from_pretrained(checkpoint, codebook_size=600)` rgrj)copypoprv from_dict)clsr~r{r|r5r6r1s zFlavaImageProcessor.from_dict)rqcCst||||||dS)N)r%r&r(r'r)r*)r )r4rdr&r(r'r)r*r5r5r6masking_generator>s z%FlavaImageProcessor.masking_generator)imagerZr[ data_formatinput_data_formatrqcKsTt|}d|ksd|kr*td||d|df}t|f||||d|S)a Resize an image to `(size["height"], size["width"])`. Args: image (`np.ndarray`): Image to resize. size (`Dict[str, int]`): Dictionary in the format `{"height": int, "width": int}` specifying the size of the output image. resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): `PILImageResampling` filter to use when resizing the image e.g. `PILImageResampling.BICUBIC`. data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format for the output image. If unset, the channel dimension format of the input image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format for the input image. If unset, the channel dimension format is inferred from the input image. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Returns: `np.ndarray`: The resized image. r/r0zFThe `size` dictionary must contain the keys `height` and `width`. Got )rZr[rr)r ValueErrorkeysr)r4rrZr[rrr{Z output_sizer5r5r6rRs#zFlavaImageProcessor.resize)rrqcCsddt|tS)Nr,r+)r)r4rr5r5r6 map_pixelsszFlavaImageProcessor.map_pixels)rrYrZr[r\r]r^r_r`rarb do_map_pixelsrrrqc Cst||| | | |||||d t|}t|r:|r:td|dkrJt|}|r`|j||||d}|rt|j|||d}|r|j|||d}| r|j || | |d}| r| |}| dk rt || |d}|S) zPreprocesses a single image.) r^r_r`rarbr\r]rYrZr[zIt looks like you are trying to rescale already rescaled images. If the input images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again.N)rrZr[r)rrZr)rZscaler)rZmeanZstdr)Zinput_channel_dim) rrrloggerZ warning_oncerrZ center_cropZrescale normalizerr)r4rrYrZr[r\r]r^r_r`rarbrrrr5r5r6_preprocess_images@  z%FlavaImageProcessor._preprocess_image)"imagesrYrZr[r\r]r^r_r`rarbrcrdr&r(r'r)r*rerfrgrhrirjrkrlrmrnrorpreturn_tensorsrrrqc"&sdk r njdk rnjtdk r8nj dk rJ nj dk r\ nj t dd dk rznjdk rnjdk rnjdk rnj dk rˆnj | dk r| nj } | dk r| nj } |dk r|nj }|dk r |nj}|dk r |nj}|dk r4|nj}|dk rH|nj}|dk r\|nj}dk rpnj dk r nj t dd dk rnjdk rnj dk r̈ nj dk rnjdk rnjtdddk rnjdk r(njdk r<njdk rPnjt|}t|spt d f dd|D}"d |"i}#|r܇ fd d|D}$|$|#d <| rj!| |||||d fd d|D}%|%|#d<t"|#|dS)a Preprocess an image or batch of images. Args: images (`ImageInput`): Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If passing in images with pixel values between 0 and 1, set `do_rescale=False`. do_resize (`bool`, *optional*, defaults to `self.do_resize`): Whether to resize the image. size (`Dict[str, int]`, *optional*, defaults to `self.size`): Size of the image. resample (`int`, *optional*, defaults to `self.resample`): Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`, Only has an effect if `do_resize` is set to `True`. do_center_crop (`bool`, *optional*, defaults to `self.do_center_crop`): Whether to center crop the image. crop_size (`Dict[str, int]`, *optional*, defaults to `self.crop_size`): Size of the center crop. Only has an effect if `do_center_crop` is set to `True`. do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): Whether to rescale the image values between [0 - 1]. rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): Rescale factor to rescale the image by if `do_rescale` is set to `True`. do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): Whether to normalize the image. image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): Image mean. image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): Image standard deviation. return_image_mask (`bool`, *optional*, defaults to `self.return_image_mask`): Whether to return the image mask. input_size_patches (`int`, *optional*, defaults to `self.input_size_patches`): Size of the patches to extract from the image. total_mask_patches (`int`, *optional*, defaults to `self.total_mask_patches`): Total number of patches to extract from the image. mask_group_min_patches (`int`, *optional*, defaults to `self.mask_group_min_patches`): Minimum number of patches to extract from the image. mask_group_max_patches (`int`, *optional*, defaults to `self.mask_group_max_patches`): Maximum number of patches to extract from the image. mask_group_min_aspect_ratio (`float`, *optional*, defaults to `self.mask_group_min_aspect_ratio`): Minimum aspect ratio of the patches to extract from the image. mask_group_max_aspect_ratio (`float`, *optional*, defaults to `self.mask_group_max_aspect_ratio`): Maximum aspect ratio of the patches to extract from the image. return_codebook_pixels (`bool`, *optional*, defaults to `self.return_codebook_pixels`): Whether to return the codebook pixels. codebook_do_resize (`bool`, *optional*, defaults to `self.codebook_do_resize`): Whether to resize the codebook pixels. codebook_size (`Dict[str, int]`, *optional*, defaults to `self.codebook_size`): Size of the codebook pixels. codebook_resample (`int`, *optional*, defaults to `self.codebook_resample`): Resampling filter to use if resizing the codebook pixels. This can be one of the enum `PILImageResampling`, Only has an effect if `codebook_do_resize` is set to `True`. codebook_do_center_crop (`bool`, *optional*, defaults to `self.codebook_do_center_crop`): Whether to center crop the codebook pixels. codebook_crop_size (`Dict[str, int]`, *optional*, defaults to `self.codebook_crop_size`): Size of the center crop of the codebook pixels. Only has an effect if `codebook_do_center_crop` is set to `True`. codebook_do_rescale (`bool`, *optional*, defaults to `self.codebook_do_rescale`): Whether to rescale the codebook pixels values between [0 - 1]. codebook_rescale_factor (`float`, *optional*, defaults to `self.codebook_rescale_factor`): Rescale factor to rescale the codebook pixels by if `codebook_do_rescale` is set to `True`. codebook_do_map_pixels (`bool`, *optional*, defaults to `self.codebook_do_map_pixels`): Whether to map the codebook pixels values. codebook_do_normalize (`bool`, *optional*, defaults to `self.codebook_do_normalize`): Whether to normalize the codebook pixels. codebook_image_mean (`float` or `List[float]`, *optional*, defaults to `self.codebook_image_mean`): Codebook pixels mean to normalize the codebook pixels by if `codebook_do_normalize` is set to `True`. codebook_image_std (`float` or `List[float]`, *optional*, defaults to `self.codebook_image_std`): Codebook pixels standard deviation to normalize the codebook pixels by if `codebook_do_normalize` is set to `True`. return_tensors (`str` or `TensorType`, *optional*): The type of tensors to return. Can be one of: - Unset: Return a list of `np.ndarray`. - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): The channel dimension format for the output image. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format for the input image. If unset, the channel dimension format is inferred from the input image. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Nr]rsrgrjzkInvalid image type. Must be of type PIL.Image.Image, numpy.ndarray, torch.Tensor, tf.Tensor or jax.ndarray.cs2g|]*} j|  ddqS)FrrYrZr[r\r]r^r_r`rarbrrrr.0img) r]rr\r`r^rYrarbrr[r_r4rZr5r6 s"z2FlavaImageProcessor.preprocess..rXcs2g|]*} j|   dqS)rrr)rjrirmrnrkrfrorprhrlrgrrr4r5r6rs"Zcodebook_pixel_values)rdr&r(r'r)r*csg|] }qSr5r5)r_)mask_generatorr5r6rsZbool_masked_pos)dataZ tensor_type)#rYrZr r[r\r]r^r_r`rarbrcrdr&r(r'r)r*rerfrgrhrkrlrirjrmrnrorprrrrr )&r4rrYrZr[r\r]r^r_r`rarbrcrdr&r(r'r)r*rerfrgrhrirjrkrlrmrnrorprrrZprocessed_imagesrZcodebook_imagesmasksr5)rjrirmrnrkrfrorprhrlrgr]rr\r`r^rYrarbrrr[r_r4rZr6 preprocesss~    "$zFlavaImageProcessor.preprocess)&rSrTrU__doc__Zmodel_input_namesrZBICUBICZLANCZOSboolrstrrAr rVrrr7 classmethodrrrr rrOZndarrayrrrZFIRSTrrrrrPILZImager __classcell__r5r5r|r6rWsfW    O    0   @   rW)4rr1r> functoolsrtypingrrrrrrr ZnumpyrOZimage_processing_utilsr r r Zimage_transformsrrZ image_utilsrrrrrrrrrrrutilsrrrrrZ get_loggerrSrrwrxryrzrrV__annotations__r rWr5r5r5r6s& $4     N