我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.FixedLengthRecordReader()。
def cifar_shuffle_batch(): batch_size = 128 num_threads = 16 # create a list of all our filenames filename_list = [data_path + 'data_batch_{}.bin'.format(i + 1) for i in range(5)] # create a filename queue # file_q = cifar_filename_queue(filename_list) file_q = tf.train.string_input_producer(filename_list) # read the data - this contains a FixedLengthRecordReader object which handles the # de-queueing of the files. It returns a processed image and label, with shapes # ready for a convolutional neural network image, label = read_data(file_q) # setup minimum number of examples that can remain in the queue after dequeuing before blocking # occurs (i.e. enqueuing is forced) - the higher the number the better the mixing but # longer initial load time min_after_dequeue = 10000 # setup the capacity of the queue - this is based on recommendations by TensorFlow to ensure # good mixing capacity = min_after_dequeue + (num_threads + 1) * batch_size # image_batch, label_batch = cifar_shuffle_queue_batch(image, label, batch_size, num_threads) image_batch, label_batch = tf.train.shuffle_batch([image, label], batch_size, capacity, min_after_dequeue, num_threads=num_threads) # now run the training cifar_run(image_batch, label_batch)
def _image_op_cifar10(filenames, relative_colors): label_bytes = 1 height = 32 width = 32 depth = 3 image_bytes = height * width * depth record_bytes = label_bytes + image_bytes filename_queue = tf.train.string_input_producer(filenames, num_epochs=1) reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) _, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]), [depth, height, width]) image = tf.transpose(depth_major, [1, 2, 0]) image = tf.cast(image, tf.float32) if relative_colors: image = util.absolute_to_relative_colors(image) return image
def _read_image(filename_queue): # copied from # https://tensorflow.googlesource.com/tensorflow/+/master/tensorflow/models/image/cifar10/cifar10_input.py # CIFAR-10 specification label_bytes = 1 height = 32 width = 32 depth = 3 image_bytes = height * width * depth record_bytes = label_bytes + image_bytes reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) _, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) label = tf.cast(tf.slice(record_bytes, [0], [label_bytes]), tf.int32) depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]), [depth, height, width]) uint8image = tf.transpose(depth_major, [1, 2, 0]) image = tf.cast(uint8image, tf.float32) return image, tf.squeeze(label)
def read_and_decode_cifar(filename_queue): label_bytes = 1 height = 32 width = 32 depth = 3 image_bytes = height * width * depth record_bytes = label_bytes + image_bytes reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) _, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]), [depth, height, width]) image = tf.transpose(depth_major, [1, 2, 0]) image = tf.cast(image, tf.float32) * (2. / 255) - 1 return image
def read_raw_images(sess, data_set): filename = ['./data/' + data_set + '_data.bin'] filename_queue = tf.train.string_input_producer(filename) print filename record_bytes = (FLAGS.height) * (FLAGS.width) * FLAGS.depth + 1 image_bytes = (FLAGS.height) * (FLAGS.width) * FLAGS.depth reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) #record_label = tf.decode_raw(value, tf.int32) tf.train.start_queue_runners(sess=sess) for i in range(0, 10): result = sess.run(record_bytes) print i, result[0], len(result) image = result[1:len(result)] print image
def read_cifar_files(filename_queue, distort_images = True): reader = tf.FixedLengthRecordReader(record_bytes=record_length) key, record_string = reader.read(filename_queue) record_bytes = tf.decode_raw(record_string, tf.uint8) image_label = tf.cast(tf.slice(record_bytes, [0], [1]), tf.int32) # Extract image image_extracted = tf.reshape(tf.slice(record_bytes, [1], [image_vec_length]), [num_channels, image_height, image_width]) # Reshape image image_uint8image = tf.transpose(image_extracted, [1, 2, 0]) reshaped_image = tf.cast(image_uint8image, tf.float32) # Randomly Crop image final_image = tf.image.resize_image_with_crop_or_pad(reshaped_image, crop_width, crop_height) if distort_images: # Randomly flip the image horizontally, change the brightness and contrast final_image = tf.image.random_flip_left_right(final_image) final_image = tf.image.random_brightness(final_image,max_delta=63) final_image = tf.image.random_contrast(final_image,lower=0.2, upper=1.8) # Normalize whitening final_image = tf.image.per_image_whitening(final_image) return(final_image, image_label) # Create a CIFAR image pipeline from reader
def _read_record(self, filename_queue): class FrameSeqRecord(object): pass record = FrameSeqRecord() record.height = self._data_img_size[0] record.width = self._data_img_size[1] record.depth = self._data_img_size[2] input_seq_length = self.input_shape[0] target_seq_length = self.target_shape[0] total_seq_length = input_seq_length + target_seq_length frame_bytes = record.height * record.width * record.depth record_bytes = frame_bytes * (total_seq_length) total_file_bytes = frame_bytes * self._serialized_sequence_length with tf.name_scope('read_record'): reader = tf.FixedLengthRecordReader(total_file_bytes) record.key, value = reader.read(filename_queue) decoded_record_bytes = tf.decode_raw(value, tf.uint8) decoded_record_bytes = tf.reshape(decoded_record_bytes, [self._serialized_sequence_length, record.height, record.width, record.depth]) # calculcate tensors [start, 0, 0, 0] rnd_start_index = tf.to_int32(tf.random_uniform([1], 0, self._serialized_sequence_length - (total_seq_length), tf.int32)) seq_start_offset = tf.SparseTensor(indices=[[0]], values=rnd_start_index, dense_shape=[4]) sequence_start = tf.sparse_tensor_to_dense(seq_start_offset) # take a random slice of frames as input record.data = tf.slice(decoded_record_bytes, sequence_start, [total_seq_length, record.height, record.width, record.depth]) return record
def data_inputs(data_dir, data_type, batch_size, runner_threads): # Input file reader filenames = input_filenames(data_dir, data_type) queue = tf.train.string_input_producer(filenames) reader = tf.FixedLengthRecordReader(record_bytes=INPUT_RECORD_BYTES) # Decode label and image _key, record_raw = reader.read(queue) record = tf.decode_raw(record_raw, tf.uint8) label = tf.cast(tf.slice(record, [0], [INPUT_LABEL_BYTES]), tf.int32) image = tf.reshape( tf.slice(record, [INPUT_LABEL_BYTES], [INPUT_IMAGE_BYTES]), [IMAGE_DEPTH, IMAGE_HEIGHT, IMAGE_WIDTH]) # Transpose image from stored DHW to HWD image_hwd = tf.transpose(image, [1, 2, 0]) # Finalize image image_float = tf.cast(image_hwd, tf.float32) if data_type == AUGMENTED_TRAINING_DATA: image_final = augmented_standardized_image(image_float) else: image_final = standardized_image(image_float) # Process image and labels using queue runner images, labels = tf.train.batch( [image_final, label], batch_size=batch_size, num_threads=runner_threads, capacity=10 * batch_size) return images, tf.reshape(labels, [batch_size])
def build_input_pipeline(input_file, batch_size): input_files = tf.train.string_input_producer([input_file]) reader = tf.FixedLengthRecordReader(record_bytes=4+4) _, raw_val_t = reader.read(input_files) int_val_t = tf.decode_raw(raw_val_t, tf.int32) center_t, context_t = int_val_t[0], int_val_t[1] center_batch_t, context_batch_t = tf.train.batch([center_t, context_t], batch_size, num_threads=4, capacity=1024) return center_batch_t, context_batch_t
def ImageProducer(filename_queue): filename = os.path.realpath(os.path.join(os.path.dirname(__file__), 'cifar10/zca.pkl')) file = open(filename, 'rb') data = pickle.load(file) file.close() Wzca= tf.constant(data['zca'],tf.float32) label_bytes = 1; height = 32; width = 32; depth = 3 image_bytes = height * width * depth record_bytes = label_bytes + image_bytes reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) label_byte_slices = tf.slice(record_bytes, [0], [label_bytes]); label = tf.cast(label_byte_slices, tf.int32) image = tf.slice(record_bytes, [label_bytes], [image_bytes])#tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]),[depth,height,width]) image = tf.cast(image, tf.float32) image = tf.reshape(image,[1,image_bytes]) image = tf.sub(image,tf.reduce_mean(image)) scale = tf.constant(55.); thresh = tf.constant(1.) std_val = tf.div(tf.sqrt(tf.reduce_sum(tf.square(image))),scale); f4 = lambda: std_val f5 = lambda: thresh normalizer = tf.cond(tf.less(std_val,1e-8),f5,f4) image = tf.div(image,normalizer) image = tf.sub(image,tf.reduce_mean(image)) img_RGB = tf.matmul(image,Wzca) depth_major = tf.reshape(img_RGB,[depth,height,width]) image = tf.transpose(depth_major, [1, 2, 0]) return image, label
def __read_cifar(filenames, shuffle=True, cifar100=False): """Reads and parses examples from CIFAR data files. """ # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. filename_queue = tf.train.string_input_producer(filenames, shuffle=shuffle,num_epochs=None) label_bytes = 1 # 2 for CIFAR-100 if cifar100: label_bytes = 2 height = 32 width = 32 depth = 3 image_bytes = height * width * depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. label = tf.cast( tf.slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]), [depth, height, width]) # Convert from [depth, height, width] to [height, width, depth]. image = tf.transpose(depth_major, [1, 2, 0]) return tf.cast(image, tf.float32), label
def read_raw_images(data_set): dirs = './data/'+data_set+'/' filename = list_binary_files(dirs) print filename filename_queue = tf.train.string_input_producer(filename) if data_set is 'train': image_bytes = FLAGS.height * FLAGS.width * FLAGS.depth record_bytes = image_bytes + 1 reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) label = tf.cast(tf.slice(record_bytes, [0], [1]), tf.int32) depth_major = tf.reshape(tf.slice(record_bytes, [1], [image_bytes]),[FLAGS.depth, FLAGS.height, FLAGS.width]) uint8image = tf.transpose(depth_major, [1, 2, 0]) return label, uint8image elif data_set is 'test': image_bytes = FLAGS.height * FLAGS.width * FLAGS.depth record_bytes = image_bytes + 1 reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) depth_major = tf.reshape(tf.slice(record_bytes, [0], [image_bytes]), [FLAGS.depth, FLAGS.height, FLAGS.width]) uint8image = tf.transpose(depth_major, [1, 2, 0]) return uint8image
def read_cifar10(filename_queue): """Reads and parses examples from CIFAR10 data files. Recommendation: if you want N-way read parallelism, call this function N times. This will give you N independent Readers reading different files & positions within those files, which will give better mixing of examples. Args: filename_queue: A queue of strings with the filenames to read from. Returns: An object representing a single example, with the following fields: height: number of rows in the result (32) width: number of columns in the result (32) depth: number of color channels in the result (3) key: a scalar string Tensor describing the filename & record number for this example. label: an int32 Tensor with the label in the range 0..9. uint8image: a [height, width, depth] uint8 Tensor with the image data """ class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. label_bytes = 1 # 2 for CIFAR-100 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast( tf.slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def read( file_pattern, batch_size, record_bytes=RECORD_BYTES, capacity=256, min_after_dequeue=128, num_threads=8, format='NCHW', normalizer=None, ): ''' Read only `sp` and `speaker` Return: `feature`: [b, c] `speaker`: [b,] ''' with tf.name_scope('InputSpectralFrame'): files = tf.gfile.Glob(file_pattern) filename_queue = tf.train.string_input_producer(files) reader = tf.FixedLengthRecordReader(record_bytes) _, value = reader.read(filename_queue) value = tf.decode_raw(value, tf.float32) value = tf.reshape(value, [FEAT_DIM,]) feature = value[:SP_DIM] # NCHW format if normalizer is not None: feature = normalizer.forward_process(feature) if format == 'NCHW': feature = tf.reshape(feature, [1, SP_DIM, 1]) elif format == 'NHWC': feature = tf.reshape(feature, [SP_DIM, 1, 1]) else: pass speaker = tf.cast(value[-1], tf.int64) return tf.train.shuffle_batch( [feature, speaker], batch_size, capacity=capacity, min_after_dequeue=min_after_dequeue, num_threads=num_threads, # enqueue_many=True, )
def read_data(file_queue): """ Data is saved in binary files. Each row has: 1st byte -> label 2nd-last byte -> 3D Volume [height, width, depth, channels] Args: file_queue -> a queue of file names saved as strings Rtns: An object with: height -> volume height width -> volume width depth -> volume depth nChan -> number of channels key -> scalar tensor with file name and record number label -> 1D int32 tensor with the associated label img3_uint8 -> 4D uint8 tensor with image data """ class record_data(object): pass img3_obj = record_data() # Dimensions of data label_bytes = 1 img3_obj.height = CFG['height'] img3_obj.width = CFG['width'] img3_obj.depth = CFG['depth'] img3_obj.nChan = CFG['nChan'] # Size in memory img3_bytes = img3_obj.height*img3_obj.width*img3_obj.depth*img3_obj.nChan record_bytes = label_bytes + img3_bytes # Read a record reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) img3_obj.key,value = reader.read(file_queue) # Convert from a string to a vector of uint8 that is record_bytes long record_bytes = tf.decode_raw(value, tf.uint8) # First byte represent the label, which we convert from uint8 -> int32 img3_obj.label = tf.cast(tf.slice(record_bytes,[0],[label_bytes]),tf.int32) # Remaining bytes after the label represent the image, which we reshape from # [depth * height * width] to [depth,height, width] depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [img3_bytes]),[img3_obj.depth,img3_obj.height,img3_obj.width, img3_obj.nChan]) img3_obj.img3_uint8 = tf.transpose(depth_major,[2,1,0,3]) return img3_obj
def read_cifar10(filename_queue): """Reads and parses examples from CIFAR10 data files. Recommendation: if you want N-way read parallelism, call this function N times. This will give you N independent Readers reading different files & positions within those files, which will give better mixing of examples. Args: filename_queue: A queue of strings with the filenames to read from. Returns: An object representing a single example, with the following fields: height: number of rows in the result (32) width: number of columns in the result (32) depth: number of color channels in the result (3) key: a scalar string Tensor describing the filename & record number for this example. label: an int32 Tensor with the label in the range 0..9. uint8image: a [height, width, depth] uint8 Tensor with the image data """ class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. label_bytes = 1 # 2 for CIFAR-100 result.height = 480 result.width = 640 result.depth = 1 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast( tf.slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def read_cifar10(filename_queue): """Reads and parses examples from CIFAR10 data files. Recommendation: if you want N-way read parallelism, call this function N times. This will give you N independent Readers reading different files & positions within those files, which will give better mixing of examples. Args: filename_queue: A queue of strings with the filenames to read from. Returns: An object representing a single example, with the following fields: height: number of rows in the result (32) width: number of columns in the result (32) depth: number of color channels in the result (3) key: a scalar string Tensor describing the filename & record number for this example. label: an int32 Tensor with the label in the range 0..9. uint8image: a [height, width, depth] uint8 Tensor with the image data """ class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. label_bytes = 1 # 2 for CIFAR-100 result.height = IMAGE_SIZE result.width = IMAGE_SIZE result.depth = 1 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast( tf.slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def read_cifar10(filename_queue): """Reads and parses examples from CIFAR10 data files. Recommendation: if you want N-way read parallelism, call this function N times. This will give you N independent Readers reading different files & positions within those files, which will give better mixing of examples. Args: filename_queue: A queue of strings with the filenames to read from. Returns: An object representing a single example, with the following fields: height: number of rows in the result (32) width: number of columns in the result (32) depth: number of color channels in the result (3) key: a scalar string Tensor describing the filename & record number for this example. label: an int32 Tensor with the label in the range 0..9. uint8image: a [height, width, depth] uint8 Tensor with the image data """ class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. label_bytes = 1 # 2 for CIFAR-100 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast( tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape( tf.strided_slice(record_bytes, [label_bytes], [label_bytes + image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def read_cifar10(filename_queue, data_format): """Reads and parses examples from CIFAR10 data files. Recommendation: if you want N-way read parallelism, call this function N times. This will give you N independent Readers reading different files & positions within those files, which will give better mixing of examples. Args: filename_queue: A queue of strings with the filenames to read from. Returns: An object representing a single example, with the following fields: height: number of rows in the result (32) width: number of columns in the result (32) depth: number of color channels in the result (3) key: a scalar string Tensor describing the filename & record number for this example. label: an int32 Tensor with the label in the range 0..9. uint8image: a [height, width, depth] uint8 Tensor with the image data """ class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. label_bytes = 1 # 2 for CIFAR-100 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast(tf.slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. # Using CHW (NCHW) as the default so no need to transpose if data_format == 'NHWC': result.uint8image = tf.transpose(depth_major, [1, 2, 0]) else: result.uint8image = depth_major return result
def read_cifar10(filename_queue, data_format): """Reads and parses examples from CIFAR10 data files. Recommendation: if you want N-way read parallelism, call this function N times. This will give you N independent Readers reading different files & positions within those files, which will give better mixing of examples. Args: filename_queue: A queue of strings with the filenames to read from. Returns: An object representing a single example, with the following fields: height: number of rows in the result (32) width: number of columns in the result (32) depth: number of color channels in the result (3) key: a scalar string Tensor describing the filename & record number for this example. label: an int32 Tensor with the label in the range 0..9. uint8image: a [height, width, depth] uint8 Tensor with the image data """ class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. label_bytes = 1 # 2 for CIFAR-100 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast(tf.slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] (NCHW) to [height, width, depth] (NHWC). if data_format == 'NHWC': result.uint8image = tf.transpose(depth_major, [1, 2, 0]) else: result.uint8image = depth_major return result
def read_cifar10(filenames, use_queue=False): class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. label_bytes = 1 # 2 for CIFAR-100 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast( tf.slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. if not reshape_to_one: depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) else: #result.uint8image = tf.cast(tf.slice(record_bytes, [label_bytes], [image_bytes]), [result.depth*result.height*result*result.width]) result.uint8image = tf.slice(record_bytes, [label_bytes], [image_bytes]) return result
def read_data(file_q): # Code from https://github.com/tensorflow/models/blob/master/tutorials/image/cifar10/cifar10_input.py class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. label_bytes = 1 # 2 for CIFAR-100 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(file_q) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast( tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape( tf.strided_slice(record_bytes, [label_bytes], [label_bytes + image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) reshaped_image = tf.cast(result.uint8image, tf.float32) height = 24 width = 24 # Image processing for evaluation. # Crop the central [height, width] of the image. resized_image = tf.image.resize_image_with_crop_or_pad(reshaped_image, height, width) # Subtract off the mean and divide by the variance of the pixels. float_image = tf.image.per_image_standardization(resized_image) # Set the shapes of tensors. float_image.set_shape([height, width, 3]) result.label.set_shape([1]) return float_image, result.label
def read_cifar10(filename_queue): """Reads and parses examples from CIFAR10 data files. Recommendation: if you want N-way read parallelism, call this function N times. This will give you N independent Readers reading different files & positions within those files, which will give better mixing of examples. Args: filename_queue: A queue of strings with the filenames to read from. Returns: An object representing a single example, with the following fields: height: number of rows in the result (32) width: number of columns in the result (32) depth: number of color channels in the result (3) key: a scalar string Tensor describing the filename & record number for this example. label: an int32 Tensor with the label in the range 0..9. uint8image: a [height, width, depth] uint8 Tensor with the image data """ class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. label_bytes = 1 # 2 for CIFAR-100 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast( tf.slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape( tf.slice(record_bytes, [label_bytes], [image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def read_fer2013(filename_queue): """Reads and parses examples from FER2013 data files. Recommendation: if you want N-way read parallelism, call this function N times. This will give you N independent Readers reading different files & positions within those files, which will give better mixing of examples. Args: filename_queue: A queue of strings with the filenames to read from. Returns: An object representing a single example, with the following fields: height: number of rows in the result (48) width: number of columns in the result (348) depth: number of color channels in the result (1) key: a scalar string Tensor describing the filename & record number for this example. label: an int32 Tensor with the label in the range 0..7. uint8image: a [height, width, depth] uint8 Tensor with the image data """ class FER2013Record(object): pass result = FER2013Record() label_bytes = 1 result.height = 48 result.width = 48 result.depth = 1 # 3 for RGB image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the FER2013 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast( tf.slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def get_raw_input_data(test_data, data_dir): """Raw CIFAR10 input data ops using the Reader ops. Args: test_data: bool, indicating if one should use the test or train set. data_dir: Path to the CIFAR-10 data directory. Returns: image: an op producing a 32x32x3 float32 image label: an op producing an int32 label """ # Verify first that we have a valid data directory if not os.path.exists(data_dir): raise ValueError("Data directory %s doesn't exist" % data_dir) # Construct a list of input file names batches_dir = os.path.join(data_dir, 'cifar-10-batches-bin') if test_data: filenames = [os.path.join(batches_dir, 'test_batch.bin')] else: filenames = [os.path.join(batches_dir, 'data_batch_%d.bin' %ii) for ii in xrange(1, 6)] # Make sure all input files actually exist for f in filenames: if not tf.gfile.Exists(f): raise ValueError('Failed to find file: ' + f) # Create a string input producer to cycle over file names filenames_queue = tf.train.string_input_producer(filenames) # CIFAR data samples are stored as contiguous labels and images label_size = 1 image_size = IMAGE_DEPTH * IMAGE_HEIGHT * IMAGE_WIDTH # Instantiate a fixed length file reader reader = tf.FixedLengthRecordReader(label_size + image_size) # Read from files key, value = reader.read(filenames_queue) record_bytes = tf.decode_raw(value, tf.uint8) # Extract label and cast to int32 label = tf.cast(tf.slice(record_bytes, [0], [label_size]), tf.int32) # Extract image and cast to float32 image = tf.cast(tf.slice(record_bytes, [label_size], [image_size]), tf.float32) # Images are stored as D x H x W vectors, but we want H x W x D # So we need to convert to a matrix image = tf.reshape(image, (IMAGE_DEPTH, IMAGE_HEIGHT, IMAGE_WIDTH)) # Transpose dimensions image = tf.transpose(image, (1, 2, 0)) return (image, label)
def read_cifar10(filename_queue): """Reads and parses examples from CIFAR10 data files. Recommendation: if you want N-way read parallelism, call this function N times. This will give you N independent Readers reading different files & positions within those files, which will give better mixing of examples. Args: filename_queue: A queue of strings with the filenames to read from. Returns: An object representing a single example, with the following fields: height: number of rows in the result (32) width: number of columns in the result (32) depth: number of color channels in the result (3) key: a scalar string Tensor describing the filename & record number for this example. label: an int32 Tensor with the label in the range 0..9. uint8image: a [height, width, depth] uint8 Tensor with the image data """ # ?????????????c?????????? class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. label_bytes = 1 # 2 for CIFAR-100 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # ????Reader???????????????? reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) # ???filename_queue????(key, value)??key?value????????tensor # ??????????????????????dequeue result.key, value = reader.read(filename_queue) # ??????????????????????????????,??????????? # ?[0, 255]??????out_type??uint8?? record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast( tf.slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def tensorflow_reader(list_0, list_1, flow_list, shuffle_all, batchs): """Average endpoint error between prediction and groundtruth Keyword arguments: list_0 -- source list of first of img pair list_1 -- source list of second of img pair flow_list -- source list of optical flow between first and second img shuffle_all -- boolean if list should be shuffled batchs -- batchsize """ assert len(list_0) == len(list_1) == len( flow_list) != 0, ('Input Lengths not correct') print("Number of inputs: " + str(len(list_0))) if shuffle_all == True: p = np.random.permutation(len(list_0)) else: p = np.arange(len(list_0)) list_0 = [list_0[i] for i in p] list_1 = [list_1[i] for i in p] flow_list = [flow_list[i] for i in p] input_queue = tf.train.slice_input_producer( [list_0, list_1], shuffle=False) # shuffled before # image reader content_0 = tf.read_file(input_queue[0]) content_1 = tf.read_file(input_queue[1]) imgs_0 = tf.image.decode_image(content_0, channels=3) imgs_1 = tf.image.decode_image(content_1, channels=3) # convert to [0, 1] images imgs_0 = tf.image.convert_image_dtype(imgs_0, dtype=tf.float32) imgs_1 = tf.image.convert_image_dtype(imgs_1, dtype=tf.float32) # flow reader filename_queue = tf.train.string_input_producer(flow_list, shuffle=False) record_bytes = FLAGS.record_bytes # 1572876 reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.float32) magic = tf.slice(record_bytes, [0], [1]) # .flo number 202021.25 size = tf.slice(record_bytes, [1], [2]) # size of flow / image flows = tf.slice(record_bytes, [3], [np.prod(FLAGS.flow_shape)]) flows = tf.reshape(flows, FLAGS.flow_shape) # set shape imgs_0.set_shape(FLAGS.img_shape) imgs_1.set_shape(FLAGS.img_shape) flows.set_shape(FLAGS.flow_shape) return tf.train.batch([imgs_0, imgs_1, flows], batch_size=batchs #,num_threads=1 )
