cmt-20230409-1: +rnn_gray

cnn_gray/resize_filter/cross_validation/new2/cnn_train1.py

@@ -15,8 +15,8 @@
 
 model_path = "model/model.ckpt"
 
-data0 = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data0 = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 data = []
 for i in range(data0.shape[0]):
     data_tmp = data0[i,:]

cnn_gray/resize_filter/cross_validation/new2/getTrainingAbnormal.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-data = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 
 print(data.shape)
 print(label.shape)

cnn_gray/resize_filter/cross_validation/new2/get_validation.py

@@ -54,5 +54,5 @@
         count = count + 1
 print(count)
 
-np.savetxt("train_set/training data.txt", data_new)
-np.savetxt("train_set/labels.txt", label_new)
+np.savetxt("train_set_new/training data.txt", data_new)
+np.savetxt("train_set_new/labels.txt", label_new)

cnn_gray/resize_filter/cross_validation/new22/cnn_train1.py

@@ -15,8 +15,8 @@
 
 model_path = "model/model.ckpt"
 
-data0 = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data0 = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 data = []
 for i in range(data0.shape[0]):
     data_tmp = data0[i,:]

cnn_gray/resize_filter/cross_validation/new22/getTrainingAbnormal.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-data = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 
 print(data.shape)
 print(label.shape)

cnn_gray/resize_filter/cross_validation/new22/get_validation.py

@@ -57,5 +57,5 @@
         count = count + 1
 print(count)
 
-np.savetxt("train_set/training data.txt", data_new)
-np.savetxt("train_set/labels.txt", label_new)
+np.savetxt("train_set_new/training data.txt", data_new)
+np.savetxt("train_set_new/labels.txt", label_new)

cnn_gray/resize_filter/cross_validation/new32/cnn_train1.py

@@ -15,8 +15,8 @@
 
 model_path = "model/model.ckpt"
 
-data0 = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data0 = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 data = []
 for i in range(data0.shape[0]):
     data_tmp = data0[i,:]

cnn_gray/resize_filter/cross_validation/new32/getTrainingAbnormal.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-data = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 
 print(data.shape)
 print(label.shape)

cnn_gray/resize_filter/cross_validation/new32/get_validation.py

@@ -54,5 +54,5 @@
         count = count + 1
 print(count)
 
-np.savetxt("train_set/training data.txt", data_new)
-np.savetxt("train_set/labels.txt", label_new)
+np.savetxt("train_set_new/training data.txt", data_new)
+np.savetxt("train_set_new/labels.txt", label_new)

cnn_gray/resize_filter/cross_validation/new42/cnn_train1.py

@@ -15,8 +15,8 @@
 
 model_path = "model/model.ckpt"
 
-data0 = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data0 = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 data = []
 for i in range(data0.shape[0]):
     data_tmp = data0[i,:]

cnn_gray/resize_filter/cross_validation/new42/getTrainingAbnormal.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-data = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 
 print(data.shape)
 print(label.shape)

cnn_gray/resize_filter/cross_validation/new42/get_validation.py

@@ -54,5 +54,5 @@
         count = count + 1
 print(count)
 
-np.savetxt("train_set/training data.txt", data_new)
-np.savetxt("train_set/labels.txt", label_new)
+np.savetxt("train_set_new/training data.txt", data_new)
+np.savetxt("train_set_new/labels.txt", label_new)

cnn_gray/resize_filter1/cross_validation/new2/cnn_train1.py

@@ -15,8 +15,8 @@
 
 model_path = "model/model.ckpt"
 
-data0 = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data0 = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 data = []
 for i in range(data0.shape[0]):
     data_tmp = data0[i,:]

cnn_gray/resize_filter1/cross_validation/new2/getTrainingAbnormal.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-data = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 
 print(data.shape)
 print(label.shape)

cnn_gray/resize_filter1/cross_validation/new2/get_validation.py

@@ -54,5 +54,5 @@
         count = count + 1
 print(count)
 
-np.savetxt("train_set/training data.txt", data_new)
-np.savetxt("train_set/labels.txt", label_new)
+np.savetxt("train_set_new/training data.txt", data_new)
+np.savetxt("train_set_new/labels.txt", label_new)

cnn_gray/resize_filter1/cross_validation/new22/cnn_train1.py

@@ -15,8 +15,8 @@
 
 model_path = "model/model.ckpt"
 
-data0 = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data0 = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 data = []
 for i in range(data0.shape[0]):
     data_tmp = data0[i,:]

cnn_gray/resize_filter1/cross_validation/new22/getTrainingAbnormal.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-data = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 
 print(data.shape)
 print(label.shape)

cnn_gray/resize_filter1/cross_validation/new22/get_validation.py

@@ -57,5 +57,5 @@
         count = count + 1
 print(count)
 
-np.savetxt("train_set/training data.txt", data_new)
-np.savetxt("train_set/labels.txt", label_new)
+np.savetxt("train_set_new/training data.txt", data_new)
+np.savetxt("train_set_new/labels.txt", label_new)

cnn_gray/resize_filter1/cross_validation/new32/cnn_train1.py

@@ -15,8 +15,8 @@
 
 model_path = "model/model.ckpt"
 
-data0 = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data0 = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 data = []
 for i in range(data0.shape[0]):
     data_tmp = data0[i,:]

cnn_gray/resize_filter1/cross_validation/new32/getTrainingAbnormal.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-data = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 
 print(data.shape)
 print(label.shape)

cnn_gray/resize_filter1/cross_validation/new32/get_validation.py

@@ -54,5 +54,5 @@
         count = count + 1
 print(count)
 
-np.savetxt("train_set/training data.txt", data_new)
-np.savetxt("train_set/labels.txt", label_new)
+np.savetxt("train_set_new/training data.txt", data_new)
+np.savetxt("train_set_new/labels.txt", label_new)

cnn_gray/resize_filter1/cross_validation/new42/cnn_train1.py

@@ -15,8 +15,8 @@
 
 model_path = "model/model.ckpt"
 
-data0 = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data0 = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 data = []
 for i in range(data0.shape[0]):
     data_tmp = data0[i,:]

cnn_gray/resize_filter1/cross_validation/new42/getTrainingAbnormal.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-data = np.genfromtxt("train_set/training data.txt")
-label = np.genfromtxt("train_set/labels.txt")
+data = np.genfromtxt("train_set_new/training data.txt")
+label = np.genfromtxt("train_set_new/labels.txt")
 
 print(data.shape)
 print(label.shape)

cnn_gray/resize_filter1/cross_validation/new42/get_validation.py

@@ -54,5 +54,5 @@
         count = count + 1
 print(count)
 
-np.savetxt("train_set/training data.txt", data_new)
-np.savetxt("train_set/labels.txt", label_new)
+np.savetxt("train_set_new/training data.txt", data_new)
+np.savetxt("train_set_new/labels.txt", label_new)

data_augmentation/data_augmentation_1.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 import numpy as np
 
-data = np.genfromtxt("train_set/training data.txt")
+data = np.genfromtxt("train_set_new/training data.txt")
 # print(data.shape)
-labels = np.genfromtxt("train_set/labels.txt")
+labels = np.genfromtxt("train_set_new/labels.txt")
 
 data_origin = []
 data_flip_a = []
@@ -121,5 +121,5 @@
 #     if labels_new[i] == 1:
 #         abnormal_count_new = abnormal_count_new + 1
 # print(abnormal_count_new)
-np.savetxt("train_set/training data.txt", data_new)
-np.savetxt("train_set/labels.txt", labels_new)
+np.savetxt("train_set_new/training data.txt", data_new)
+np.savetxt("train_set_new/labels.txt", labels_new)

rnn_gray/.gitignore

@@ -0,0 +1,12 @@
+.idea
+**/*.txt
+**/chapters/02*
+**/chapters/03*
+**/chapters/04*
+**/chapters/05*
+**/chapters/07*
+**/chapters/08*
+**/errata
+**/img/book_cover.jpg
+**/tensorflowbook-master/README.md
+**/model

rnn_gray/BidirectionalSequenceLabellingModel.py

@@ -0,0 +1,97 @@
+import tensorflow as tf
+
+from helpers import lazy_property
+
+
+class BidirectionalSequenceLabellingModel:
+
+    def __init__(self, data, target, params):
+        self.data = data
+        self.target = target
+        self.params = params
+        self.prediction
+        self.cost
+        self.error
+        self.optimize
+
+    @lazy_property
+    def length(self):
+        used = tf.sign(tf.reduce_max(tf.abs(self.data), reduction_indices=2))
+        length = tf.reduce_sum(used, reduction_indices=1)
+        length = tf.cast(length, tf.int32)
+        return length
+
+    @lazy_property
+    def prediction(self):
+        output = self._bidirectional_rnn(self.data, self.length)
+        num_classes = int(self.target.get_shape()[2])
+        prediction = self._shared_softmax(output, num_classes)
+        return prediction
+
+    def _bidirectional_rnn(self, data, length):
+        length_64 = tf.cast(length, tf.int64)
+        forward, _ = tf.nn.dynamic_rnn(
+            cell=self.params.rnn_cell(self.params.rnn_hidden),
+            inputs=data,
+            dtype=tf.float32,
+            sequence_length=length,
+            scope='rnn-forward')
+        backward, _ = tf.nn.dynamic_rnn(
+            cell=self.params.rnn_cell(self.params.rnn_hidden),
+            inputs=tf.reverse_sequence(data, length_64, seq_dim=1),
+            dtype=tf.float32,
+            sequence_length=self.length,
+            scope='rnn-backward')
+        backward = tf.reverse_sequence(backward, length_64, seq_dim=1)
+        output = tf.concat(2, [forward, backward])
+        return output
+
+    def _shared_softmax(self, data, out_size):
+        max_length = int(data.get_shape()[1])
+        in_size = int(data.get_shape()[2])
+        weight = tf.Variable(tf.truncated_normal(
+            [in_size, out_size], stddev=0.01))
+        bias = tf.Variable(tf.constant(0.1, shape=[out_size]))
+        # Flatten to apply same weights to all time steps.
+        flat = tf.reshape(data, [-1, in_size])
+        output = tf.nn.softmax(tf.matmul(flat, weight) + bias)
+        output = tf.reshape(output, [-1, max_length, out_size])
+        return output
+
+    @lazy_property
+    def cost(self):
+        # Compute cross entropy for each frame.
+        cross_entropy = self.target * tf.log(self.prediction)
+        cross_entropy = -tf.reduce_sum(cross_entropy, reduction_indices=2)
+        mask = tf.sign(tf.reduce_max(tf.abs(self.target), reduction_indices=2))
+        cross_entropy *= mask
+        # Average over actual sequence lengths.
+        cross_entropy = tf.reduce_sum(cross_entropy, reduction_indices=1)
+        cross_entropy /= tf.cast(self.length, tf.float32)
+        return tf.reduce_mean(cross_entropy)
+
+    @lazy_property
+    def error(self):
+        mistakes = tf.not_equal(
+            tf.argmax(self.target, 2), tf.argmax(self.prediction, 2))
+        mistakes = tf.cast(mistakes, tf.float32)
+        mask = tf.sign(tf.reduce_max(tf.abs(self.target), reduction_indices=2))
+        mistakes *= mask
+        # Average over actual sequence lengths.
+        mistakes = tf.reduce_sum(mistakes, reduction_indices=1)
+        mistakes /= tf.cast(self.length, tf.float32)
+        return tf.reduce_mean(mistakes)
+
+    @lazy_property
+    def optimize(self):
+        gradient = self.params.optimizer.compute_gradients(self.cost)
+        try:
+            limit = self.params.gradient_clipping
+            gradient = [
+                (tf.clip_by_value(g, -limit, limit), v)
+                if g is not None else (None, v)
+                for g, v in gradient]
+        except AttributeError:
+            print('No gradient clipping parameter specified.')
+        optimize = self.params.optimizer.apply_gradients(gradient)
+        return optimize