sid,eld,sidd dataset介绍,dng图像处理

文章目录

- SID dataset
- - 1. SID dataset 概述
  - 2. SID 读取和显示代码
  - 3. 一些示例
- SIDD dataset
- ELD dataset
- DNG camera pipeline

SID dataset

1. SID dataset 概述

SID 是Learning to See in the Dark 论文中提出的暗光raw数据集

其中包括两个相机的拍摄数据 Sony alpha7S II 和 Fujifilm X-T2.
下面主要介绍下sony camera的数据

拍摄照度：
室外 0.2-5 lux
室内 0.03-0.3 lux

一共包括short和long两个文件夹：
long文件夹共231个ARW文件，其中20208__00_10s.ARW表示的是场景20208，第00次拍摄，曝光时间是10s
在这里插入图片描述

short文件夹对应的短曝光的拍摄图像, 一共2697张图像
short的曝光时间设置为参考图像的 1/100, 1/250, 1/300。比如参考groundtruth 曝光是10s, 则short图像的曝光可能被设置为 0.1s, 0.04s, 0.033s。相同设置可能被拍摄多张，可以用来开发 burst multiframe相关算法。
在这里插入图片描述

2. SID 读取和显示代码

利用rawpy 库
在这里插入图片描述

import glob
import os

import matplotlib.pyplot as plt
import numpy as np
import rawpy

import colour
from colour_demosaicing import demosaicing_CFA_Bayer_Menon2007
def pack_raw(raw):
    # pack Bayer image to 4 channels
    im = raw.raw_image_visible.astype(np.float32)
    im = np.maximum(im - 512, 0) / (16383 - 512)  # subtract the black level

    im = np.expand_dims(im, axis=2)
    img_shape = im.shape
    H = img_shape[0]
    W = img_shape[1]

    out = np.concatenate((im[0:H:2, 0:W:2, :],
                          im[0:H:2, 1:W:2, :],
                          im[1:H:2, 1:W:2, :],
                          im[1:H:2, 0:W:2, :]), axis=2)
    return out


def pack_raw_bayer(raw):
    # 和上面的函数功能一样，都是减去black level，然后分成4 channnel图像
    # pack Bayer image to 4 channels
    im = raw.raw_image_visible.astype(np.float32)
    raw_pattern = raw.raw_pattern
    R = np.where(raw_pattern == 0)
    G1 = np.where(raw_pattern == 1)
    B = np.where(raw_pattern == 2)
    G2 = np.where(raw_pattern == 3)

    white_point = 16383
    img_shape = im.shape
    H = img_shape[0]
    W = img_shape[1]

    out = np.stack((im[R[0][0]:H:2, R[1][0]:W:2],  # RGBG
                    im[G1[0][0]:H:2, G1[1][0]:W:2],
                    im[B[0][0]:H:2, B[1][0]:W:2],
                    im[G2[0][0]:H:2, G2[1][0]:W:2]), axis=0).astype(np.float32)

    black_level = np.array(raw.black_level_per_channel)[:, None, None].astype(np.float32)

    out = (out - black_level) / (white_point - black_level)
    out = np.clip(out, 0, 1)

    return out
if __name__ == "__main__":
    input_dir = r'D:\dataset\ELD\sid\Sony\Sony\short'
    gt_dir = r'D:\dataset\ELD\sid\Sony\Sony\long'

    train_id = 1
    in_files = glob.glob(os.path.join(input_dir, '%05d_00*.ARW' % train_id))
    gt_files = glob.glob(os.path.join(gt_dir, '%05d_00*.ARW' % train_id))
    print(in_files, gt_files)

    in_path = in_files[0]
    gt_path = gt_files[0]
    print(in_path, gt_path)

    # 获取曝光时间
    in_fn = os.path.basename(in_path)
    gt_fn = os.path.basename(gt_path)
    in_exposure = float(in_fn[9:-5])
    gt_exposure = float(gt_fn[9:-5])
    ratio = min(gt_exposure / in_exposure, 300)
    print('exp time:', in_exposure, gt_exposure, ratio)

    raw = rawpy.imread(in_path)
    gt_raw = rawpy.imread(gt_path)
    print('raw meta info :', raw.black_level_per_channel, raw.raw_pattern)

    im = raw.raw_image_visible.astype(np.float32)
    im = np.maximum(im - 512, 0) / (16383 - 512) * ratio # subtract the black level
    im1 = demosaicing_CFA_Bayer_Menon2007(im, 'RGGB')
    im11 = raw.postprocess(use_camera_wb=True, half_size=False, no_auto_bright=True, output_bps=8)

    im2 = gt_raw.postprocess(use_camera_wb=True, half_size=False, no_auto_bright=True, output_bps=8)

    # im1 raw数据乘上一个ratio, 然后 demosaicing, 显示绿色的raw图
    # im11 对raw数据 应用isp, 由于曝光时间短, 没有乘上ratio, 可能显示sRGB 全黑
    # im2 是groundtruth 参考图，曝光充分，正常显示sRGB图
    plt.figure()
    plt.subplot(131)
    plt.imshow(im1)
    plt.subplot(132)
    plt.imshow(im11)
    plt.subplot(133)
    plt.imshow(im2)
    plt.show()

    # noise image
    im11 = im11 / im11.max()
    im11 = im11 * ratio
    im11 = np.clip(im11, 0, 1)
    plt.figure()
    plt.imshow(im11)
    plt.show()

    # process all
    gt_files = glob.glob(os.path.join(gt_dir, '*.ARW'))
    for file in gt_files:
        print(file)
        gt_raw = rawpy.imread(file)
        im2 = gt_raw.postprocess(use_camera_wb=True, half_size=False, no_auto_bright=True, output_bps=8)
        cv2.imwrite(file[:-4] + '.png', im2[..., ::-1])

3. 一些示例

在这里插入图片描述

gt 和 noise:
在这里插入图片描述

将中间的黑色图乘上ratio显示noise image(只是为了显示，这样操作是不对的):
在这里插入图片描述

SIDD dataset

link:https://www.eecs.yorku.ca/~kamel/sidd/dataset.php
论文 A High-Quality Denoising Dataset for Smartphone Cameras 提出的一个数据集

利用多张有噪声图像，利用对齐技术和fusion技术，生成无噪声图像作为ground truth.

官方网站上有 small , medium, full三个版本的数据集。我下载了medium的数据集， full的太大内存。

medium 版本包含raw 和 sRGB两个文件夹
在这里插入图片描述

命名方式：

<scene-instance-number>_<scene_number>_<smartphone-code>_<ISO-level>_<shutter-speed>_<illuminant-temperature>_<illuminant-brightness-code>

raw:
160 x (2 + 2) = 640 张图像
sRGB:
对应raw的 640张 sRGB 图像。
下载的数据集中有详细说明。

对应的mat数据， wb, ccm, gamma后的图像如下：
在这里插入图片描述

code:

import colour_demosaicing
import scipy.io as sio
import glob
import os

import cv2
import matplotlib.pyplot as plt
import numpy as np
import rawpy

import colour
from colour_demosaicing import demosaicing_CFA_Bayer_Menon2007
import h5py


def extract_metainfo(path='0151_METADATA_RAW_010.MAT'):
    meta = sio.loadmat(path)['metadata']
    mat_vals = meta[0][0]
    mat_keys = mat_vals.dtype.descr

    keys = []
    for item in mat_keys:
        keys.append(item[0])

    py_dict = {}
    for key in keys:
        py_dict[key] = mat_vals[key]

    device = py_dict['Model'][0].lower()
    bitDepth = py_dict['BitDepth'][0][0]
    if 'iphone' in device or bitDepth != 16:
        noise = py_dict['UnknownTags'][-2][0][-1][0][:2]
        iso = py_dict['DigitalCamera'][0, 0]['ISOSpeedRatings'][0][0]
        pattern = py_dict['SubIFDs'][0][0]['UnknownTags'][0][0][1][0][-1][0]
        time = py_dict['DigitalCamera'][0, 0]['ExposureTime'][0][0]

    else:
        noise = py_dict['UnknownTags'][-1][0][-1][0][:2]
        iso = py_dict['ISOSpeedRatings'][0][0]
        pattern = py_dict['UnknownTags'][1][0][-1][0]
        time = py_dict['ExposureTime'][0][0]  # the 0th row and 0th line item

    rgb = ['R', 'G', 'B']
    pattern = ''.join([rgb[i] for i in pattern])

    asShotNeutral = py_dict['AsShotNeutral'][0]
    b_gain, _, r_gain = asShotNeutral

    # only load ccm1
    ccm = py_dict['ColorMatrix1'][0].astype(float).reshape((3, 3))

    return {'device': device,
            'pattern': pattern,
            'iso': iso,
            'noise': noise,
            'time': time,
            'wb': np.array([r_gain, 1, b_gain]),
            'ccm': ccm, }


def extract_metainfo2(file):
    meta = sio.loadmat(file)['metadata']
    mat_vals = meta[0][0]
    mat_keys = mat_vals.dtype.descr

    keys = []
    for item in mat_keys:
        keys.append(item[0])

    py_dict = {}
    for key in keys:
        py_dict[key] = mat_vals[key]

    return py_dict


def fix_orientation(image, orientation):
    # 1 = Horizontal(normal)
    # 2 = Mirror horizontal
    # 3 = Rotate 180
    # 4 = Mirror vertical
    # 5 = Mirror horizontal and rotate 270 CW
    # 6 = Rotate 90 CW
    # 7 = Mirror horizontal and rotate 90 CW
    # 8 = Rotate 270 CW

    if type(orientation) is list:
        orientation = orientation[0]

    if orientation == 1:
        pass
    elif orientation == 2:
        image = cv2.flip(image, 0)
    elif orientation == 3:
        image = cv2.rotate(image, cv2.ROTATE_180)
    elif orientation == 4:
        image = cv2.flip(image, 1)
    elif orientation == 5:
        image = cv2.flip(image, 0)
        image = cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE)
    elif orientation == 6:
        image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
    elif orientation == 7:
        image = cv2.flip(image, 0)
        image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
    elif orientation == 8:
        image = cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE)

    return image

def process_mat_img(file, py_dict, pattern='bggr'):
    '''
    :param file: 输入 mat 文件
    :return: srgb image
    '''
    data = {}
    f = h5py.File(file)
    for k, v in f.items():
        data[k] = np.array(v)
    data0 = data['x']  # [1000:2000,2000:3000]
    data = fix_orientation(data0, py_dict['Orientation'])

    rgb = colour_demosaicing.demosaicing_CFA_Bayer_Menon2007(data, pattern)

    wb_gain = 1 / py_dict['AsShotNeutral']
    wb_gain = wb_gain.astype(np.float32).reshape(-1, 1, 3)
    rgb_wb = rgb * wb_gain
    rgb_wb = np.clip(rgb_wb, 0, 1)

    xyz2cam1 = np.reshape(np.asarray(py_dict['ColorMatrix1']),
                          (3, 3))  # 不同光源的标定矩阵，这里xyz2cam1应该是D65， 对应py_dict['CalibrationIlluminant1']
    xyz2cam2 = np.reshape(np.asarray(py_dict['ColorMatrix2']), (3, 3))
    # normalize rows (needed?)
    xyz2cam1 = xyz2cam1 / np.sum(xyz2cam1, axis=1, keepdims=True)
    xyz2cam2 = xyz2cam2 / np.sum(xyz2cam1, axis=1, keepdims=True)
    # inverse
    cam2xyz1 = np.linalg.inv(xyz2cam1)
    cam2xyz2 = np.linalg.inv(xyz2cam2)
    # for now, use one matrix  # TODO: interpolate btween both
    rgb_xyz = rgb_wb.reshape(-1, 3) @ cam2xyz1.T
    rgb_xyz = rgb_xyz.reshape(rgb_wb.shape)
    rgb_xyz = np.clip(rgb_xyz, 0.0, 1.0)
    xyz2srgb = np.array([[3.2404542, -1.5371385, -0.4985314],
                         [-0.9692660, 1.8760108, 0.0415560],
                         [0.0556434, -0.2040259, 1.0572252]])

    # normalize rows (needed?)
    rgb_ccm = rgb_xyz.reshape(-1, 3) @ xyz2srgb.T
    rgb_ccm = rgb_ccm.reshape(rgb_wb.shape)
    rgb_ccm = np.clip(rgb_ccm, 0.0, 1.0)

    rgb_gamma = rgb_ccm ** (1 / 2.2)
    rgb_gamma = np.clip(rgb_gamma, 0, 1)

    # rgb_gamma_save = np.clip(rgb_gamma * 255, 0, 255).astype(np.uint8)
    # cv2.imwrite('dd.png', rgb_gamma_save[::-1, :, ::-1])

    plt.figure()
    plt.subplot(221)
    plt.imshow(rgb)
    plt.subplot(222)
    plt.imshow(rgb_wb)
    plt.subplot(223)
    plt.imshow(rgb_ccm)
    plt.subplot(224)
    plt.imshow(rgb_gamma)
    plt.show()

if __name__ == "__main__":
    file1 = r'D:\dataset\SIDD_Medium_Raw_Parts\SIDD_Medium_Raw\Data\0055_003_N6_00800_01000_5500_N\0055_GT_RAW_010.MAT'
    file2 = r'D:\dataset\SIDD_Medium_Raw_Parts\SIDD_Medium_Raw\Data\0055_003_N6_00800_01000_5500_N\0055_GT_RAW_011.MAT'

    file3 = r'D:\dataset\SIDD_Medium_Raw_Parts\SIDD_Medium_Raw\Data\0055_003_N6_00800_01000_5500_N\0055_METADATA_RAW_010.MAT'
    file4 = r'D:\dataset\SIDD_Medium_Raw_Parts\SIDD_Medium_Raw\Data\0055_003_N6_00800_01000_5500_N\0055_METADATA_RAW_011.MAT'

    file5 = r'D:\dataset\SIDD_Medium_Raw_Parts\SIDD_Medium_Raw\Data\0055_003_N6_00800_01000_5500_N\0055_NOISY_RAW_010.MAT'
    file6 = r'D:\dataset\SIDD_Medium_Raw_Parts\SIDD_Medium_Raw\Data\0055_003_N6_00800_01000_5500_N\0055_NOISY_RAW_011.MAT'

    metainfo = extract_metainfo(file3)
    print(metainfo)
    py_dict = extract_metainfo2(file3)

    # isp: wb, ccm, gamma
    process_mat_img(file5, py_dict, metainfo['pattern'])
    print('py_dict info:', py_dict)
    print(py_dict['AsShotNeutral'], py_dict['ColorMatrix1'], py_dict['ColorMatrix2'])
    print(py_dict['CalibrationIlluminant1'], py_dict['CalibrationIlluminant2'])
    print(py_dict['Orientation'])
    print(py_dict['Height'], py_dict['Width'], py_dict['BitDepth'])