有效地读取图像，对比opencv、PIL、turbojpeg、lmdb、tfrecords

import cv2
from turbojpeg  TurboJPEG,TJPF_GRAY,TJSAMP_GRAY,TJFLAG_PROGRESSIVE
# using default library installation
jpeg = TurboJPEG()

 decoding input.jpg to BGR array
in_file = open('input.jpg',rb')
bgr_array = jpeg.decode(in_file.read())
in_file.close()
cv2.imshow(bgr_array,bgr_array)
cv2.waitKey(0)

 -*- coding: utf-8 -*-
 lmdb
  
 如果train文件夹下没有data.mbd或lock.mdb文件，则会生成一个空的，如果有，不会覆盖
# map_size定义最大储存容量，单位是kb，以下定义1TB容量
env = lmdb.open("./train"，map_size=1099511627776)
env.close()

 os
from argparse  ArgumentParser

 lmdb
 numpy as np

from tools  get_images_paths


def store_many_lmdb(images_list,save_path):

    num_images = len(images_list)   number of images in our folder

    file_sizes = [os.path.getsize(item) for item in images_list]   all file sizes
    max_size_index = np.argmax(file_sizes)   the maximum file size index

     maximum database size in bytes
    map_size = num_images * cv2.imread(images_list[max_size_index]).nbytes * 10

    env = lmdb.open(save_path,map_size=map_size)   create lmdb environment

    with env.begin(write=True) as txn:   start writing to environment
        for i,image in enumerate(images_list):
            with open(image,") as file:
                data = file.read()   read image as bytes
                key = f{i:08}"   get image key
                txn.put(key.encode(ascii"),data)   put the key-value into database

    env.close()   close the environment


if __name__ == __main__:
    parser = ArgumentParser()
    parser.add_argument(
        --path-pstr,required=True,help=path to the images folder to collect--output-opath to the output environment directory file i.e. "path/to/folder/env/" parser.parse_args()
    if not os.path.exists(args.output):
        os.makedirs(args.output)

    images = get_images_paths(args.path)
    store_many_lmdb(images,args.output)

 tensorflow as tf

 _byte_feature(value):
    """Convert string / byte into bytes_list."""
    if isinstance(value,type(tf.constant(0))):
        value = value.numpy()   BytesList can't unpack string from EagerTensor.
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))


 _int64_feature(value):
    Convert bool / enum / int / uint into int64_list.return tf.train.Feature(int64_list=tf.train.Int64List(value= image_example(image_string,label):
    feature = {
        label: _int64_feature(label),1)">image_raw: _byte_feature(image_string),}
    return tf.train.Example(features=tf.train.Features(feature=feature))


 store_many_tfrecords(images_list,save_file):

    assert save_file.endswith(
        .tfrecordsFile path is wrong,it should contain "*myname*.tfrecords"

    directory = os.path.dirname(save_file)
     os.path.exists(directory):
        os.makedirs(directory)

    with tf.io.TFRecordWriter(save_file) as writer:   start writer
        for label,filename in enumerate(images_list):   cycle by each image path
            image_string = open(filename,1)">").read()   read the image as bytes string
            tf_example = image_example(
                image_string,label,)   save the data as tf.Example object
            writer.write(tf_example.SerializeToString())   and write it into database


path to the output tfrecords file i.e. "path/to/folder/myname.tfrecords" parser.parse_args()
    image_paths = get_images_paths(args.path)
    store_many_tfrecords(image_paths,args.output)

from abc  abstractmethod
from timeit  default_timer as timer

 numpy as np
 tensorflow as tf
from PIL  Image
 TurboJPEG

os.environ[TF_CPP_MIN_LOG_LEVEL"] = 3"


class ImageLoader:
    extensions: tuple = (.png",1)">.jpg.jpeg.tiff.bmp.gif)

    def __init__(self,path: str,mode: str = BGR):
        self.path = path
        self.mode = mode
        self.dataset = self.parse_input(self.path)
        self.sample_idx = 0

     parse_input(self,path):

         single image or tfrecords file
         os.path.isfile(path):
             path.lower().endswith(
                self.extensions,fUnsupportable extension,please,use one of {self.extensions}"
            return [path]

         os.path.isdir(path):
             lmdb environment
            if any([file.endswith(.mdb") for file  os.listdir(path)]):
                 path
            else:
                 folder with images
                paths = [os.path.join(path,image) for image  os.listdir(path)]
                 paths

    __iter__(self):
        self.sample_idx = 0
         self

    __len__(self):
         len(self.dataset)

    @abstractmethod
    __next__pass


 CV2Loader(ImageLoader):
    (self):
        start = timer()
        path = self.dataset[self.sample_idx]   get image path by index from the dataset
        image = cv2.imread(path)   read the image
        full_time = timer() - start
        if self.mode == RGB:
            start = timer()
            image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)   change color mode
            full_time += timer() - start
        self.sample_idx += 1
         image,full_time


 PILLoader(ImageLoader):
     get image path by index from the dataset
        image = np.asarray(Image.open(path))   read the image as numpy array
        full_time = timer() - TurboJpegLoader(ImageLoader):
    kwargs):
        super(TurboJpegLoader,self).__init__(path,1)">kwargs)
        self.jpeg_reader = TurboJPEG()   create TurboJPEG object for image reading

     timer()
        file = open(self.dataset[self.sample_idx],1)">")   open the input file as bytes
        full_time = timer() -:
            mode =elif self.mode == :
            mode = 1
        start = timer()
        image = self.jpeg_reader.decode(file.read(),mode)   decode raw image
        full_time += timer() - LmdbLoader(ImageLoader):
    kwargs):
        super(LmdbLoader,1)">kwargs)
        self.path = path
        self._dataset_size = 0
        self.dataset = self.open_database()

     we need to open the database to read images from it
     open_database(self):
        lmdb_env = lmdb.open(self.path)   open the environment by path
        lmdb_txn = lmdb_env.begin()   start reading
        lmdb_cursor = lmdb_txn.cursor()   create cursor to iterate through the database
        self._dataset_size = lmdb_env.stat()[
            entries
        ]   get number of items in full dataset
         lmdb_cursor

    (self):
        self.dataset.first()   return the cursor to the first database element
         timer()
        raw_image = self.dataset.value()   get raw image
        image = np.frombuffer(raw_image,dtype=np.uint8)   convert it to numpy
        image = cv2.imdecode(image,cv2.IMREAD_COLOR)   decode image
        full_time = timer() - timer()
            image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
            full_time += timer() - start
        start = timer()
        self.dataset.next()   step to the next element in database
        full_time += timer() -return self._dataset_size   get dataset length


 TFRecordsLoader(ImageLoader):
    kwargs):
        super(TFRecordsLoader,1)">kwargs)
        self._dataset = open_database(self):
         _parse_image_function(example_proto):
             tf.io.parse_single_example(example_proto,image_feature_description)

         dataset structure description
        image_feature_description = {
            : tf.io.FixedLenFeature([],tf.int64),tf.string),}
        raw_image_dataset = tf.data.TFRecordDataset(self.path)   open dataset by path
        parsed_image_dataset = raw_image_dataset.map(
            _parse_image_function,1)"> parse dataset using structure description

         parsed_image_dataset

    (self):
        self.dataset = self._dataset.as_numpy_iterator()
         timer()
        value = next(self.dataset)[
             step to the next element in database and get new image
        image = tf.image.decode_jpeg(value).numpy()   decode raw image
        full_time = timer() - self._dataset.reduce(
            np.int64(0),lambda x,_: x + 1 get dataset length


methods = {
    cv2: CV2Loader,1)">pil: PILLoader,1)">turbojpeg: TurboJpegLoader,1)">lmdb: LmdbLoader,1)">tfrecords: TFRecordsLoader,}

 cv2

from loader  (
    CV2Loader,LmdbLoader,PILLoader,TFRecordsLoader,TurboJpegLoader,methods,)


 show_image(method,image):
    cv2.imshow(f{method} imageif k == 27:   check ESC pressing
         True
    :
         False


 show_images(loader):
    num_images = len(loader)
    loader = iter(loader)
    for idx  range(num_images):
        image,time = next(loader)
        print_info(image,time)
        stop = show_image(type(loader).__name__ stop:
            cv2.destroyAllWindows()
            return


 print_info(image,time):
    print(
        fImage with {image.shape[0]}x{image.shape[1]} size has been loading for {time} seconds demo(method,path):
    loader = methods[method](path)   get the image loader
    show_images(loader)


 ArgumentParser()

    parser.add_argument(
        path to image,folder of images,lmdb environment path or tfrecords database path--method],1)">Image loading methods to use in benchmark parser.parse_args()

    demo(args.method,args.path)