Python tensorflow 模块，read_file() 实例源码

def _abspath_no_label_load_file(path, epochs=None, shuffle=True, seed=0):
    filename_queue = tf.train.string_input_producer([path],
            num_epochs=epochs, shuffle=shuffle, seed=seed)

    reader = tf.TextLineReader()
    key, value = reader.read(filename_queue)
    #image_path, = tf.decode_csv(value, record_defaults=[['']], field_delim=' ')
    image_path = value

    image_abspath = image_path

    image_content = tf.read_file(image_abspath)
    image = decode_image(image_content, channels=3)
    image.set_shape([None, None, 3])

    imgshape = tf.shape(image)[:2]

    return image, imgshape, image_path

def read_tensor_from_image_file(file_name='test.jpg', input_height=128, input_width=128,
                input_mean=0, input_std=255):


  input_name = "file_reader"
  output_name = "normalized"
  file_reader = tf.read_file(file_name, input_name)
  image_reader = tf.image.decode_jpeg(file_reader, channels = 3, name='jpeg_reader')
  float_caster = tf.cast(image_reader, tf.float32)
  dims_expander = tf.expand_dims(float_caster, 0);
  resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width])
  normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])
  sess = tf.Session()
  result = sess.run(normalized)

  return result

def image_reading(path: str, resized_size: Tuple[int, int]=None, data_augmentation: bool=False,
                  padding: bool=False) -> Tuple[tf.Tensor, tf.Tensor]:
    # Read image
    image_content = tf.read_file(path, name='image_reader')
    image = tf.cond(tf.equal(tf.string_split([path], '.').values[1], tf.constant('jpg', dtype=tf.string)),
                    true_fn=lambda: tf.image.decode_jpeg(image_content, channels=1, try_recover_truncated=True), # TODO channels = 3 ?
                    false_fn=lambda: tf.image.decode_png(image_content, channels=1), name='image_decoding')

    # Data augmentation
    if data_augmentation:
        image = augment_data(image)

    # Padding
    if padding:
        with tf.name_scope('padding'):
            image, img_width = padding_inputs_width(image, resized_size, increment=CONST.DIMENSION_REDUCTION_W_POOLING)
    # Resize
    else:
        image = tf.image.resize_images(image, size=resized_size)
        img_width = tf.shape(image)[1]

    with tf.control_dependencies([tf.assert_equal(image.shape[:2], resized_size)]):
        return image, img_width

def read_image_and_label(image_label_q):
    # Returns three Tensors: the decoded PNG image, the hour, and the minute.
    filename, hour_str, minute_str = tf.decode_csv(
        image_label_q.dequeue(), [[""], [""], [""]], " ")
    file_contents = tf.read_file(filename)

    # Decode image from PNG, and cast it to a float.
    example = tf.image.decode_png(file_contents, channels=image_channels)
    image = tf.cast(example, tf.float32)

    # Set the tensor size manually from the image.
    image.set_shape([image_size, image_size, image_channels])

    # Do per-image whitening (zero mean, unit standard deviation). Without this,
    # the learning algorithm diverges almost immediately because the gradient is
    # too big.
    image = tf.image.per_image_whitening(image)

    # The label should be an integer.
    hour = tf.string_to_number(hour_str, out_type=tf.int32)
    minute = tf.string_to_number(minute_str, out_type=tf.int32)

    return image, hour, minute

def tf_ops(self, capacity=32, produce_filenames=False):
        im_path, label_path = tf.train.slice_input_producer(
            [tf.constant(self.images), tf.constant(self.labels)],
            capacity=capacity,
            shuffle=self.shuffle)
        im = tf.read_file(im_path)
        im = tf.image.decode_image(im, channels=3)
        im = tf.cast(im, tf.float32)
        im.set_shape((1024, 2048, 3))
        label = tf.read_file(label_path)
        label = tf.image.decode_image(label, channels=1)
        label = label[:, :, 0]
        label = tf.cast(label, tf.int32)
        label.set_shape((1024, 2048))
        if produce_filenames:
            return im, label, im_path, label_path
        else:
            return im, label

def read_tensor_from_image_file(file_name, input_height=299, input_width=299,
                input_mean=0, input_std=255):
  input_name = "file_reader"
  output_name = "normalized"
  file_reader = tf.read_file(file_name, input_name)
  if file_name.endswith(".png"):
    image_reader = tf.image.decode_png(file_reader, channels = 3,
                                       name='png_reader')
  elif file_name.endswith(".gif"):
    image_reader = tf.squeeze(tf.image.decode_gif(file_reader,
                                                  name='gif_reader'))
  elif file_name.endswith(".bmp"):
    image_reader = tf.image.decode_bmp(file_reader, name='bmp_reader')
  else:
    image_reader = tf.image.decode_jpeg(file_reader, channels = 3,
                                        name='jpeg_reader')
  float_caster = tf.cast(image_reader, tf.float32)
  dims_expander = tf.expand_dims(float_caster, 0);
  resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width])
  normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])
  sess = tf.Session()
  result = sess.run(normalized)

  return result

def decode(self, filename, distort_data, whiten_data = True):
        """distort: random distort the iamge"""
        image_tensor = tf.read_file(filename)
        image_tensor = self.decode_fun(image_tensor, channels = self.channels, ratio = self.ratio)
        image_tensor = tf.image.convert_image_dtype(image_tensor, tf.float32)
        image_tensor = tf.image.resize_images(image_tensor, 
                                        [self.shape[0] + self.offset, self.shape[1] + self.offset])

        if distort_data:
                # it will crop in the function
                image_tensor = self.distort_op(image_tensor)
        else:
                image_tensor = tf.image.resize_image_with_crop_or_pad(image_tensor,
                                                                                self.shape[0], self.shape[1])
        if whiten_data:
                # Subtract off the mean and divide by the variance of the pixels.
                image_tensor = tf.image.per_image_whitening(image_tensor)

        return image_tensor

def decode(self, filename, distort_data, whiten_data = True):
        """distort: random distort the iamge"""
        image_tensor = tf.read_file(filename)
        image_tensor = self.decode_fun(image_tensor, channels = self.channels, ratio = self.ratio)
        image_tensor = tf.image.convert_image_dtype(image_tensor, tf.float32)
        image_tensor = tf.image.resize_images(image_tensor, 
                                        [self.shape[0] + self.offset, self.shape[1] + self.offset])

        if distort_data:
                # it will crop in the function
                image_tensor = self.distort_op(image_tensor)
        else:
                image_tensor = tf.image.resize_image_with_crop_or_pad(image_tensor,
                                                                                self.shape[0], self.shape[1])
        if whiten_data:
                # Subtract off the mean and divide by the variance of the pixels.
                image_tensor = tf.image.per_image_whitening(image_tensor)

        return image_tensor

def read_image(self, image_path):
        # tf.decode_image does not return the image size, this is an ugly workaround to handle both jpeg and png
        path_length = string_length_tf(image_path)[0]
        file_extension = tf.substr(image_path, path_length - 3, 3)
        file_cond = tf.equal(file_extension, 'jpg')

        image  = tf.cond(file_cond, lambda: tf.image.decode_jpeg(tf.read_file(image_path)), lambda: tf.image.decode_png(tf.read_file(image_path)))

        # if the dataset is cityscapes, we crop the last fifth to remove the car hood
        if self.dataset == 'cityscapes':
            o_height    = tf.shape(image)[0]
            crop_height = (o_height * 4) / 5
            image  =  image[:crop_height,:,:]

        image  = tf.image.convert_image_dtype(image,  tf.float32)
        image  = tf.image.resize_images(image,  [self.params.height, self.params.width], tf.image.ResizeMethod.AREA)

        return image

def read_data(prefix, labels_dic, mixing, files_from_cl):
    image_list = sorted(map(lambda x: os.path.join(prefix, x),
                        filter(lambda x: x.endswith('JPEG'), files_from_cl)))
    prefix2 = []

    for file_i in image_list:
        tmp = file_i.split(prefix+'/')[1].split("_")[0]
        prefix2.append(tmp)

    prefix2     = np.array(prefix2)
    labels_list = np.array([mixing[labels_dic[i]] for i in prefix2])

    assert(len(image_list) == len(labels_list))
    images             = tf.convert_to_tensor(image_list, dtype=tf.string)
    labels             = tf.convert_to_tensor(labels_list, dtype=tf.int32)
    input_queue        = tf.train.slice_input_producer([images, labels], shuffle=True, capacity=2000)
    image_file_content = tf.read_file(input_queue[0])
    label              = input_queue[1]
    image              = tf.image.resize_images(tf.image.decode_jpeg(image_file_content, channels=3), [256, 256])
    image              = tf.random_crop(image, [224, 224, 3])
    image              = tf.image.random_flip_left_right(image)

    return image, label

def read_tensor_from_image_file(file_name, input_height=299, input_width=299,
                input_mean=0, input_std=255):
  input_name = "file_reader"
  output_name = "normalized"
  file_reader = tf.read_file(file_name, input_name)
  if file_name.endswith(".png"):
    image_reader = tf.image.decode_png(file_reader, channels = 3,
                                       name='png_reader')
  elif file_name.endswith(".gif"):
    image_reader = tf.squeeze(tf.image.decode_gif(file_reader,
                                                  name='gif_reader'))
  elif file_name.endswith(".bmp"):
    image_reader = tf.image.decode_bmp(file_reader, name='bmp_reader')
  else:
    image_reader = tf.image.decode_jpeg(file_reader, channels = 3,
                                        name='jpeg_reader')
  float_caster = tf.cast(image_reader, tf.float32)
  dims_expander = tf.expand_dims(float_caster, 0);
  resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width])
  normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])
  sess = tf.Session()
  result = sess.run(normalized)

  return result

def get_feature_vectors_from_files(self, image_paths):
        # Decode image
        with self.graph.as_default():
            image_path = tf.placeholder(tf.string, None, 'image_path')
            image = tf.image.decode_jpeg(tf.read_file(image_path))

        # Extract features
        features = []
        with tf.Session(graph=self.graph) as sess:
            for path in image_paths:
                image_data = sess.run(
                    image,
                    {image_path: path}
                )
                feature_data = sess.run(
                    self.feature_op,
                    {INPUT_DATA_TENSOR_NAME: image_data}
                )
                features.append(feature_data)
        return features

def verify_image_jpeg(imagepath, imageshape):
    scope = inspect.stack()[0][3]
    try:
        graph = tf.get_default_graph()
        path = graph.get_tensor_by_name(scope + '/path:0')
        decode = graph.get_tensor_by_name(scope + '/decode_jpeg:0')
    except KeyError:
        tf.logging.debug('creating decode_jpeg tensor')
        path = tf.placeholder(tf.string, name=scope + '/path')
        imagefile = tf.read_file(path, name=scope + '/read_file')
        decode = tf.image.decode_jpeg(imagefile, channels=3, name=scope + '/decode_jpeg')
    try:
        image = tf.get_default_session().run(decode, {path: imagepath})
    except:
        return False
    return np.all(np.equal(image.shape[:2], imageshape[:2]))

def decode_image_objects(paths):
    with tf.name_scope(inspect.stack()[0][3]):
        with tf.name_scope('parse_example'):
            reader = tf.TFRecordReader()
            _, serialized = reader.read(tf.train.string_input_producer(paths))
            example = tf.parse_single_example(serialized, features={
                'imagepath': tf.FixedLenFeature([], tf.string),
                'imageshape': tf.FixedLenFeature([3], tf.int64),
                'objects': tf.FixedLenFeature([2], tf.string),
            })
        imagepath = example['imagepath']
        objects = example['objects']
        with tf.name_scope('decode_objects'):
            objects_class = tf.decode_raw(objects[0], tf.int64, name='objects_class')
            objects_coord = tf.decode_raw(objects[1], tf.float32)
            objects_coord = tf.reshape(objects_coord, [-1, 4], name='objects_coord')
        with tf.name_scope('load_image'):
            imagefile = tf.read_file(imagepath)
            image = tf.image.decode_jpeg(imagefile, channels=3)
    return image, example['imageshape'], objects_class, objects_coord

def _voc_seg_load_file(path, epochs=None, shuffle=True, seed=0):

    PASCAL_ROOT = os.environ['VOC_DIR']
    filename_queue = tf.train.string_input_producer([path],
            num_epochs=epochs, shuffle=shuffle, seed=seed)

    reader = tf.TextLineReader()
    key, value = reader.read(filename_queue)
    image_path, seg_path = tf.decode_csv(value, record_defaults=[[''], ['']], field_delim=' ')

    image_abspath = PASCAL_ROOT + image_path
    seg_abspath = PASCAL_ROOT + seg_path

    image_content = tf.read_file(image_abspath)
    image = decode_image(image_content, channels=3)
    image.set_shape([None, None, 3])

    imgshape = tf.shape(image)[:2]
    imgname = image_path

    seg_content = tf.read_file(seg_abspath)
    seg = tf.cast(tf.image.decode_png(seg_content, channels=1), tf.int32)
    return image, seg, imgshape, imgname

def _imagenet_load_file(path, epochs=None, shuffle=True, seed=0, subset='train', prepare_path=True):
    IMAGENET_ROOT = os.environ.get('IMAGENET_DIR', '')
    if not isinstance(path, list):
        path = [path]
    filename_queue = tf.train.string_input_producer(path,
            num_epochs=epochs, shuffle=shuffle, seed=seed)

    reader = tf.TextLineReader()
    key, value = reader.read(filename_queue)
    image_path, label_str = tf.decode_csv(value, record_defaults=[[''], ['']], field_delim=' ')

    if prepare_path:
        image_abspath = IMAGENET_ROOT + '/images/' + subset + image_path
    else:
        image_abspath = image_path

    image_content = tf.read_file(image_abspath)
    image = decode_image(image_content, channels=3)
    image.set_shape([None, None, 3])

    imgshape = tf.shape(image)[:2]
    label = tf.string_to_number(label_str, out_type=tf.int32)

    return image, label, imgshape, image_path

def _relpath_no_label_load_file(path, root_path, epochs=None, shuffle=True, seed=0):
    filename_queue = tf.train.string_input_producer([path],
            num_epochs=epochs, shuffle=shuffle, seed=seed)

    reader = tf.TextLineReader()
    key, value = reader.read(filename_queue)
    #image_path, = tf.decode_csv(value, record_defaults=[['']], field_delim=' ')
    image_path = value

    image_abspath = root_path + '/' + image_path

    image_content = tf.read_file(image_abspath)
    image = decode_image(image_content, channels=3)
    image.set_shape([None, None, 3])

    imgshape = tf.shape(image)[:2]

    return image, imgshape, image_path

def CamVid_reader_seq(filename_queue, seq_length):
  image_seq_filenames = tf.split(axis=0, num_or_size_splits=seq_length, value=filename_queue[0])
  label_seq_filenames = tf.split(axis=0, num_or_size_splits=seq_length, value=filename_queue[1])

  image_seq = []
  label_seq = []
  for im ,la in zip(image_seq_filenames, label_seq_filenames):
    imageValue = tf.read_file(tf.squeeze(im))
    labelValue = tf.read_file(tf.squeeze(la))
    image_bytes = tf.image.decode_png(imageValue)
    label_bytes = tf.image.decode_png(labelValue)
    image = tf.cast(tf.reshape(image_bytes, (IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_DEPTH)), tf.float32)
    label = tf.cast(tf.reshape(label_bytes, (IMAGE_HEIGHT, IMAGE_WIDTH, 1)), tf.int64)
    image_seq.append(image)
    label_seq.append(label)
  return image_seq, label_seq

def _process(self):

        def img_process(fn):
            img = tf.image.decode_image(tf.read_file(fn))
            cropped = tf.image.resize_image_with_crop_or_pad(img, tf.app.flags.FLAGS.crop_height, tf.app.flags.FLAGS.crop_width)
            new_img = tf.image.resize_images(cropped, (tf.app.flags.FLAGS.target_height, tf.app.flags.FLAGS.target_width), method = 
                                             tf.image.ResizeMethod.AREA)
            return fn, new_img

        filenames = tf.constant(glob.glob(os.path.join(self.src_dir,"*")))
        dataset = tf.data.Dataset.from_tensor_slices((filenames, ))
        dataset = dataset.map(img_process)
        dataset = dataset.shuffle(buffer_size=10000)
        dataset = dataset.batch(tf.app.flags.FLAGS.batch_size)
        dataset = dataset.repeat(tf.app.flags.FLAGS.epochs)

        iterator = dataset.make_one_shot_iterator()

        labels, imgs = iterator.get_next()
        return labels, imgs

def _process(self):

        def img_process(fn):
            img = tf.image.decode_image(tf.read_file(fn))
            cropped = tf.image.resize_image_with_crop_or_pad(img, tf.app.flags.FLAGS.crop_height, tf.app.flags.FLAGS.crop_width)
            new_img = tf.image.resize_images(cropped, (tf.app.flags.FLAGS.target_height, tf.app.flags.FLAGS.target_width), method = 
                                             tf.image.ResizeMethod.AREA)
            return fn, new_img

        filenames = tf.constant(glob.glob(os.path.join(self.src_dir,"*")))
        dataset = tf.data.Dataset.from_tensor_slices((filenames, ))
        dataset = dataset.map(img_process)
        dataset = dataset.shuffle(buffer_size=10000)
        dataset = dataset.batch(tf.app.flags.FLAGS.batch_size)
        dataset = dataset.repeat(tf.app.flags.FLAGS.epochs)

        iterator = dataset.make_one_shot_iterator()

        labels, imgs = iterator.get_next()
        return labels, imgs

def _process(self):

        def img_process(fn):
            img = tf.image.decode_image(tf.read_file(fn))
            cropped = tf.image.resize_image_with_crop_or_pad(img, tf.app.flags.FLAGS.crop_height, tf.app.flags.FLAGS.crop_width)
            new_img = tf.image.resize_images(cropped, (tf.app.flags.FLAGS.target_height, tf.app.flags.FLAGS.target_width), method = 
                                             tf.image.ResizeMethod.AREA)
            return fn, new_img

        filenames = tf.constant(glob.glob(os.path.join(self.src_dir,"*")))
        dataset = tf.data.Dataset.from_tensor_slices((filenames, ))
        dataset = dataset.map(img_process)
        dataset = dataset.shuffle(buffer_size=10000)
        dataset = dataset.batch(tf.app.flags.FLAGS.batch_size)
        dataset = dataset.repeat(tf.app.flags.FLAGS.epochs)

        iterator = dataset.make_one_shot_iterator()

        labels, imgs = iterator.get_next()
        return labels, imgs

def _get_image(self):
        _, records = self.reader.read(self.input_queue)
        file_names = tf.decode_csv(records, [tf.constant([], tf.string), tf.constant([], tf.string)],
                                   field_delim=None, name=None)

        im_raw = tf.read_file(self.base_folder+file_names[0])
        seg_raw = tf.read_file(self.base_folder+file_names[1])
        image = tf.reshape(
                            tf.cast(tf.image.decode_png(
                                                    im_raw,
                                                    channels=1, dtype=tf.uint16),
                                    tf.float32), self.image_size, name='input_image')
        seg = tf.reshape(
                        tf.cast(tf.image.decode_png(
                                                    seg_raw,
                                                    channels=1, dtype=tf.uint8),
                                tf.float32), self.image_size, name='input_seg')

        return image, seg, file_names[0]

def _get_image(self):

        im_filename = tf.sparse_tensor_to_dense(tf.string_split(tf.expand_dims(self.raw_queue.dequeue(), 0), ':'), '')
        im_filename.set_shape([1, 2])
        im_raw = tf.read_file(self.base_folder+im_filename[0][0])
        seg_raw = tf.read_file(self.base_folder+im_filename[0][1])

        image = tf.reshape(tf.cast(tf.image.decode_png(im_raw, channels=1, dtype=tf.uint16), tf.float32),
                           self.image_size, name='input_image')
        seg = tf.reshape(tf.cast(tf.image.decode_png(seg_raw, channels=1, dtype=tf.uint8), tf.float32), self.image_size,
                         name='input_seg')
        if self.partial_frame:
            crop_y_start = int(((1-self.partial_frame) * self.image_size[0])/2)
            crop_y_end = int(((1+self.partial_frame) * self.image_size[0])/2)
            crop_x_start = int(((1-self.partial_frame) * self.image_size[1])/2)
            crop_x_end = int(((1+self.partial_frame) * self.image_size[1])/2)
            image = tf.slice(image, [crop_y_start, crop_x_start, 0], [crop_y_end, crop_x_end, -1])
            seg = tf.slice(seg, [crop_y_start, crop_x_start, 0], [crop_y_end, crop_x_end, -1])

        return image, seg, im_filename[0][0], im_filename[0][1]

def _get_image_sequence(self):
        filenames = self.raw_queue
        im_list = []
        seg_list = []
        for i in range(0, len(filenames), 2):
            im_filename, seg_filename = filenames[i], filenames[i+1]
            im_raw = tf.read_file(self.base_folder+im_filename)
            seg_raw = tf.read_file(self.base_folder+seg_filename)

            image_size = self.image_size + (1, )
            image = tf.reshape(tf.cast(tf.image.decode_png(im_raw, channels=1, dtype=tf.uint16), tf.float32),
                               image_size)
            seg = tf.reshape(tf.cast(tf.image.decode_png(seg_raw, channels=1, dtype=tf.uint8), tf.float32),
                             image_size)
            if self.partial_frame:
                crop_y_start = int(((1-self.partial_frame) * image_size[0])/2)
                crop_y_end = int(((1+self.partial_frame) * image_size[0])/2)
                crop_x_start = int(((1-self.partial_frame) * image_size[1])/2)
                crop_x_end = int(((1+self.partial_frame) * image_size[1])/2)
                image = tf.slice(image, [crop_y_start, crop_x_start, 0], [crop_y_end, crop_x_end, -1])
                seg = tf.slice(seg, [crop_y_start, crop_x_start, 0], [crop_y_end, crop_x_end, -1])
            im_list.append(image)
            seg_list.append(seg)

        return im_list, seg_list, filenames

def subtract_mean_multi(image_tensors, mean_image_path, channels=NUM_CHANNELS, image_size=512):

    mean_image = tf.convert_to_tensor(mean_image_path, dtype=tf.string)
    mean_file_contents = tf.read_file(mean_image)
    mean_uint8 = tf.image.decode_png(mean_file_contents, channels=channels)
    mean_uint8.set_shape([image_size, image_size, channels])


    images_mean_free = []
    for image_tensor in image_tensors:
        image_tensor.set_shape([image_size, image_size, channels])
        image = tf.cast(image_tensor, tf.float32)

        #subtract mean image
        image_mean_free = tf.subtract(image, tf.cast(mean_uint8, tf.float32))
        images_mean_free.append(image_mean_free)

    return images_mean_free

def __init__(self, checkpoint, threads):
        # Create the session
        self.session = tf.Session(graph = tf.Graph(), config=tf.ConfigProto(inter_op_parallelism_threads=threads,
                                                                            intra_op_parallelism_threads=threads))

        with self.session.graph.as_default():
            # Construct the model
            self.images = tf.placeholder(tf.float32, [None, self.HEIGHT, self.WIDTH, 3])

            with tf_slim.arg_scope(tf_slim.nets.resnet_v1.resnet_arg_scope(is_training=False)):
                resnet, _ = tf_slim.nets.resnet_v1.resnet_v1_50(self.images, num_classes = self.CLASSES)

            self.predictions = tf.argmax(tf.squeeze(resnet, [1, 2]), 1)

            # Load the checkpoint
            self.saver = tf.train.Saver()
            self.saver.restore(self.session, checkpoint)

            # JPG loading
            self.jpeg_file = tf.placeholder(tf.string, [])
            self.jpeg_data = tf.image.resize_image_with_crop_or_pad(tf.image.decode_jpeg(tf.read_file(self.jpeg_file), channels=3), self.HEIGHT, self.WIDTH)

def read_image(self, image_path):
        # tf.decode_image does not return the image size, this is an ugly workaround to handle both jpeg and png
        path_length = string_length_tf(image_path)[0]
        file_extension = tf.substr(image_path, path_length - 3, 3)
        file_cond = tf.equal(file_extension, 'jpg')

        image  = tf.cond(file_cond, lambda: tf.image.decode_jpeg(tf.read_file(image_path)), lambda: tf.image.decode_png(tf.read_file(image_path)))

        # if the dataset is cityscapes, we crop the last fifth to remove the car hood
        if self.dataset == 'cityscapes':
            o_height    = tf.shape(image)[0]
            crop_height = (o_height * 4) // 5
            image  =  image[:crop_height,:,:]

        image  = tf.image.convert_image_dtype(image,  tf.float32)
        image  = tf.image.resize_images(image,  [self.params.height, self.params.width], tf.image.ResizeMethod.AREA)

        return image

def input_pipeline(self, batch_size, num_epochs=None, aug=False):
        images_tensor = tf.convert_to_tensor(self.image_names, dtype=tf.string)
        labels_tensor = tf.convert_to_tensor(self.labels, dtype=tf.int64)
        input_queue = tf.train.slice_input_producer([images_tensor, labels_tensor], num_epochs=num_epochs)

        labels = input_queue[1]
        images_content = tf.read_file(input_queue[0])
        images = tf.image.convert_image_dtype(tf.image.decode_png(images_content, channels=1), tf.float32)
        if aug:
            images = self.data_augmentation(images)
        new_size = tf.constant([FLAGS.image_size, FLAGS.image_size], dtype=tf.int32)
        images = tf.image.resize_images(images, new_size)
        image_batch, label_batch = tf.train.shuffle_batch([images, labels], batch_size=batch_size, capacity=50000,
                                                          min_after_dequeue=10000)
        # print 'image_batch', image_batch.get_shape()
        return image_batch, label_batch

def dataset_reader(filename_queue): #prev name: CamVid_reader

    image_filename = filename_queue[0] #tensor of type string
    label_filename = filename_queue[1] #tensor of type string

    #get png encoded image
    imageValue = tf.read_file(image_filename)
    labelValue = tf.read_file(label_filename)

    #decodes a png image into a uint8 or uint16 tensor
    #returns a tensor of type dtype with shape [height, width, depth]
    image_bytes = tf.image.decode_png(imageValue)
    label_bytes = tf.image.decode_png(labelValue) #Labels are png, not jpeg

    image = tf.reshape(image_bytes, (FLAGS.image_h, FLAGS.image_w, FLAGS.image_c))
    label = tf.reshape(label_bytes, (FLAGS.image_h, FLAGS.image_w, 1))

    return image, label

def read_images_from_disk(input_queue):
    """Consumes a single filename and label as a ' '-delimited string.
    Args:
      filename_and_label_tensor: A scalar string tensor.
    Returns:
      Two tensors: the decoded image, and the string label.
    """
    label = input_queue[1]
    file_contents = tf.read_file(input_queue[0])
    example = tf.image.decode_png(file_contents, channels=3)
    return example, label


# def random_rotate_image(image):
#     angle = np.random.uniform(low=-10.0, high=10.0)
#     return misc.imrotate(image, angle, 'bicubic')

def __read_imagenet(path, shuffle=True, save_file = 'imagenet_files.csv'):
    if not os.path.exists(save_file):
        def class_index(fn):
            class_id = re.search(r'(n\d+)', fn).group(1)
            return synset_map[class_id]['index']

        file_list = glob.glob(path+'/*/*.JPEG')
        label_indexes = []
        with open(save_file, 'wb') as csv_file:
            wr = csv.writer(csv_file, quoting=csv.QUOTE_NONE)
            for f in file_list:
                idx = class_index(f)
                label_indexes.append(idx)
                wr.writerow([f, idx])

    with open(save_file, 'rb') as f:
        reader = csv.reader(f)
        file_list = list(reader)
    file_tuple, label_tuple = zip(*file_list)

    filename, labels = tf.train.slice_input_producer([list(file_tuple), list(label_tuple)], shuffle=shuffle)
    images = tf.image.decode_jpeg(tf.read_file(filename), channels=3)
    images = tf.div(tf.add(tf.to_float(images), -127), 128)
    return images, tf.string_to_number(labels, tf.int32)

def read_tensor_from_image_file(self, image_file):
        input_name = "file_reader"
        file_reader = tf.read_file(image_file, input_name)
        if image_file.endswith(".png"):
            image_reader = tf.image.decode_png(file_reader, channels=3,
                                               name='png_reader')
        elif image_file.endswith(".gif"):
            image_reader = tf.squeeze(tf.image.decode_gif(file_reader,
                                                          name='gif_reader'))
        elif image_file.endswith(".bmp"):
            image_reader = tf.image.decode_bmp(file_reader, name='bmp_reader')
        else:
            image_reader = tf.image.decode_jpeg(file_reader, channels=3,
                                                name='jpeg_reader')

        image_preprocessing_fn = preprocessing_factory.get_preprocessing(
            self.model_name, is_training=False)
        processed_image = image_preprocessing_fn(image_reader, 224, 224)
        processed_images = tf.expand_dims(processed_image, 0)
        sess = tf.Session()
        im_result = sess.run(processed_images)
        return im_result

def read_tensor_from_image_file(file_name, input_height=299, input_width=299,
                input_mean=0, input_std=255):
  input_name = "file_reader"
  output_name = "normalized"
  file_reader = tf.read_file(file_name, input_name)
  if file_name.endswith(".png"):
    image_reader = tf.image.decode_png(file_reader, channels = 3,
                                       name='png_reader')
  elif file_name.endswith(".gif"):
    image_reader = tf.squeeze(tf.image.decode_gif(file_reader,
                                                  name='gif_reader'))
  elif file_name.endswith(".bmp"):
    image_reader = tf.image.decode_bmp(file_reader, name='bmp_reader')
  else:
    image_reader = tf.image.decode_jpeg(file_reader, channels = 3,
                                        name='jpeg_reader')
  float_caster = tf.cast(image_reader, tf.float32)
  dims_expander = tf.expand_dims(float_caster, 0);
  resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width])
  normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])
  sess = tf.Session()
  result = sess.run(normalized)

  return result

def read_images_from_disk(input_queue):
    """Consumes a single filename and label as a ' '-delimited string.
    Args:
      filename_and_label_tensor: A scalar string tensor.
    Returns:
      Two tensors: the decoded image, and the string label.
    """
    label = input_queue[1]
    file_contents = tf.read_file(input_queue[0])
    example = tf.image.decode_image(file_contents, channels=3)
    return example, label

def read_bbbc006(all_files_queue):
    """Reads and parses examples from BBBC006 data files.
    Recommendation: if you want N-way read parallelism, call this function
    N times.  This will give you N independent Readers reading different
    files & positions within those files, which will give better mixing of
    examples.

    Args:
        filename_queue: A queue of strings with the filenames to read from.
    Returns:
        An object representing a single example, with the following fields:
            label: a [height, width, 2] uint8 Tensor with contours tensor in depth 0 and
                segments tensor in depth 1.
            uint8image: a [height, width, depth] uint8 Tensor with the image data
    """

    class BBBC006Record(object):
        pass

    result = BBBC006Record()

    # Read a record, getting filenames from the filename_queue.
    text_reader = tf.TextLineReader()
    _, csv_content = text_reader.read(all_files_queue)

    i_path, c_path, s_path = tf.decode_csv(csv_content,
                                           record_defaults=[[""], [""], [""]])

    result.uint8image = read_from_queue(tf.read_file(i_path))
    contour = read_from_queue(tf.read_file(c_path))
    segment = read_from_queue(tf.read_file(s_path))

    result.label = tf.concat([contour, segment], 2)
    return result

def read_images_from_disk(input_queue, input_size, random_scale): 
    """Read one image and its corresponding mask with optional pre-processing.

    Args:
      input_queue: tf queue with paths to the image and its mask.
      input_size: a tuple with (height, width) values.
                  If not given, return images of original size.
      random_scale: whether to randomly scale the images prior
                    to random crop.

    Returns:
      Two tensors: the decoded image and its mask.
    """
    img_contents = tf.read_file(input_queue[0])
    label_contents = tf.read_file(input_queue[1])
    shape = input_queue[2]

    img = tf.image.decode_jpeg(img_contents, channels=3)
    label = tf.image.decode_png(label_contents, channels=1)
    if input_size is not None:
        h, w = input_size
        if random_scale:
            scale = tf.random_uniform([1], minval=0.75, maxval=1.25, dtype=tf.float32, seed=None)
            h_new = tf.to_int32(tf.multiply(tf.to_float(tf.shape(img)[0]), scale))
            w_new = tf.to_int32(tf.multiply(tf.to_float(tf.shape(img)[1]), scale))
            new_shape = tf.squeeze(tf.stack([h_new, w_new]), axis=[1])
            img = tf.image.resize_images(img, new_shape)
            label = tf.image.resize_nearest_neighbor(tf.expand_dims(label, 0), new_shape)
            label = tf.squeeze(label, axis=[0]) # resize_image_with_crop_or_pad accepts 3D-tensor.
        img = tf.image.resize_image_with_crop_or_pad(img, h, w)
        label = tf.image.resize_image_with_crop_or_pad(label, h, w)
    # RGB -> BGR.
    img_r, img_g, img_b = tf.split(axis=2, num_or_size_splits=3, value=img)
    img = tf.cast(tf.concat(axis=2, values=[img_b, img_g, img_r]), dtype=tf.float32)
    # Extract mean.
    img -= IMG_MEAN

    return img, label, shape

def load_image(self, image_path, is_jpeg):
        # Read the file
        file_data = tf.read_file(image_path)
        # Decode the image data
        img = tf.cond(
            is_jpeg,
            lambda: tf.image.decode_jpeg(file_data, channels=self.data_spec.channels),
            lambda: tf.image.decode_png(file_data, channels=self.data_spec.channels))
        if self.data_spec.expects_bgr:
            # Convert from RGB channel ordering to BGR
            # This matches, for instance, how OpenCV orders the channels.
            img = tf.reverse(img, [False, False, True])
        return img

def process(self):
    idx, image_path = self.path_queue.dequeue()
    img = tf.image.decode_jpeg(tf.read_file(image_path), channels=3) # It is an RGB PNG
    img = tf.reverse(img, [False, False, True]) # RGB -> BGR
    return (idx, ImageReader.process_single_image(img, self.image_spec['scale_size'], 
                                               self.image_spec['crop_size'], 
                                               self.image_spec['mean']))

def read_images_and_labels_from_disk(input_queue):
    """ Reads images and labels from disk and sets their shape

      Args:
        input_queue: created by slice_input_producer
    """
    image_contents = tf.read_file(input_queue[0])
    label_contents = tf.read_file(input_queue[1])
    image = tf.image.decode_png(image_contents, IMAGE_CHANNELS)
    label = tf.image.decode_png(label_contents, LABEL_CHANNELS)
    image.set_shape([IMAGE_HEIGHT_ORIG, IMAGE_WIDTH_ORIG, IMAGE_CHANNELS])
    label.set_shape([IMAGE_HEIGHT_ORIG, IMAGE_WIDTH_ORIG, LABEL_CHANNELS])
    print("Image has dtype", image.dtype, "and shape", image.get_shape(), "after decoding from disk.")
    print("Label has dtype", label.dtype, "and shape", label.get_shape(), "after decoding from disk.")
    return image, label

def tf_ops(self, capacity=32):
        im_path, label_path = tf.train.slice_input_producer(
            [tf.constant(self.images), tf.constant(self.labels)],
            capacity=capacity,
            shuffle=self.shuffle)
        im_shape = [1024, 1024 + self.overlap, 3]
        label_shape = [1024, 1024 + self.overlap]
        queue = tf.FIFOQueue(capacity, [tf.float32, tf.int32],
                             shapes=[im_shape, label_shape])
        im = tf.read_file(im_path)
        im = tf.image.decode_image(im, channels=3)
        im = tf.cast(im, tf.float32)
        left_im = im[:, :1024 + self.overlap, :]
        right_im = im[:, 1024 - self.overlap:, :]
        left_im.set_shape(im_shape)
        right_im.set_shape(im_shape)
        label = tf.read_file(label_path)
        label = tf.image.decode_image(label, channels=1)
        label = label[:, :, 0]
        label = tf.cast(label, tf.int32)
        label_pad = tf.ones([1024, self.overlap], dtype=tf.int32) * 255
        left_label = tf.concat([label[:, :1024], label_pad], 1)
        right_label = tf.concat([label_pad, label[:, 1024:]], 1)
        left_label.set_shape(label_shape)
        right_label.set_shape(label_shape)
        ims = tf.stack([left_im, right_im], 0)
        labels = tf.stack([left_label, right_label], 0)
        enqueue_op = queue.enqueue_many([ims, labels])
        qr = tf.train.QueueRunner(queue, [enqueue_op])
        tf.train.add_queue_runner(qr)
        return queue.dequeue()

def tf_ops(self, capacity=32):
        images = ops.convert_to_tensor(self._image_fn_list, dtype=dtypes.string)
        labels = ops.convert_to_tensor(self._label_list, dtype=dtypes.int32)

        # Makes an input queue
        im_fn_q, labl_q = tf.train.slice_input_producer(
            [images, labels], capacity=capacity, shuffle=True)

        file_contents_q = tf.read_file(im_fn_q)
        im_q = self._decoder(file_contents_q, channels=3)

        return im_q, labl_q

def mp3_tensors_from_directory(directory, batch_size, channels=2, format='mp3', seconds=30, bitrate=16384):
      filenames = glob.glob(directory+"/**/*."+format)
      labels,total_labels = build_labels(sorted(glob.glob(directory+"/*")))
      num_examples_per_epoch = 10000

      # Create a queue that produces the filenames to read.
      classes = [labels[f.split('/')[-2]] for f in filenames]
      print("Found files", len(filenames))

      filenames = tf.convert_to_tensor(filenames, dtype=tf.string)
      classes = tf.convert_to_tensor(classes, dtype=tf.int32)
      print("[0]", filenames[0], classes[0])

      input_queue = tf.train.slice_input_producer([filenames, classes])

      # Read examples from files in the filename queue.
      print("INPUT_QUEUE", input_queue[0])
      value = tf.read_file(input_queue[0])
      #preprocess = tf.read_file(input_queue[0]+'.preprocess')

      print("Preloaded data", value)
      #print("Loaded data", data)

      label = input_queue[1]

      min_fraction_of_examples_in_queue = 0.4
      min_queue_examples = int(num_examples_per_epoch *
                               min_fraction_of_examples_in_queue)

      #data = tf.cast(data, tf.float32)
      data = ffmpeg.decode_audio(value, file_format=format, samples_per_second=bitrate, channel_count=channels)
      data = shared.resize_audio_patch.resize_audio_with_crop_or_pad(data, seconds*bitrate*channels, 0,True)
      #data = tf.slice(data, [0,0], [seconds*bitrate, channels])
      tf.Tensor.set_shape(data, [seconds*bitrate, channels])
      #data = tf.minimum(data, 1)
      #data = tf.maximum(data, -1)
      data = data/tf.reduce_max(tf.reshape(tf.abs(data),[-1]))
      print("DATA IS", data)
      x,y=_get_data(data, label, min_queue_examples, batch_size)

      return x, y, total_labels, num_examples_per_epoch

def read_image_jpg(image_path, depth=3, scale=True):
    """Reads the image from image_path (tf.string tensor) [jpg image].
    Cast the result to float32 and if scale=True scale it in [-1,1]
    using scale_image. Otherwise the values are in [0,1]
    Reuturn:
        the decoded jpeg image, casted to float32
    """

    image = tf.image.convert_image_dtype(
        tf.image.decode_jpeg(tf.read_file(image_path), channels=depth),
        dtype=tf.float32)
    if scale:
        image = scale_image(image)
    return image

def read_image_png(image_path, depth=3, scale=True):
    """Reads the image from image_path (tf.string tensor) [jpg image].
    Cast the result to float32 and if scale=True scale it in [-1,1]
    using scale_image. Otherwise the values are in [0,1]
    Reuturn:
        the decoded jpeg image, casted to float32
    """
    image = tf.image.convert_image_dtype(
        tf.image.decode_png(tf.read_file(image_path), channels=depth),
        dtype=tf.float32)
    if scale:
        image = scale_image(image)
    return image

def read_dataset(self, queue, input_size, data_format):
        image = tf.image.decode_jpeg(
            tf.read_file(queue[0]), channels=3, name=self.scope+'/image')
        label = tf.image.decode_png(
            tf.read_file(queue[1]), channels=1, name=self.scope+'/label')
        image = tf.image.resize_images(image, input_size)
        label = tf.image.resize_images(label, input_size, 1)
        if data_format == 'NCHW':
            self.channel_axis = 1
            image = tf.transpose(image, [2, 0, 1])
            label = tf.transpose(label, [2, 0, 1])
        image -= tf.reduce_mean(tf.cast(image, dtype=tf.float32),
                                (0, 1), name=self.scope+'/mean')
        return image, label

def read_my_file_format(self, filename_queue):
        reader = tf.TextLineReader()
        key, record_string = reader.read(filename_queue)
        # "a" means representative value to indicate type for csv cell value.
        image_file_name, depth_file_name = tf.decode_csv(record_string, [["a"], ["a"]])

        image_png_data = tf.read_file(image_file_name)
        depth_png_data = tf.read_file(depth_file_name)
        # channels=1 means image is read as gray-scale
        image_decoded = tf.image.decode_png(image_png_data, channels=1)
        image_decoded.set_shape([512, 512, 1])
        depth_decoded = tf.image.decode_png(depth_png_data, channels=1)
        depth_decoded.set_shape([512, 512, 1])
        return image_decoded, depth_decoded