working codes

Author: chen0040

README.md

@@ -1,2 +1,46 @@
 # java-word-embedding
+
 Word embedding in Java
+
+The current project provides GloVe word embedding that developer can directly use within their project.
+
+# Usage
+
+The [sample codes](src/main/java/com/github/chen0040/embeddings/GloVeModelDemo.java) below shows how to use
+[GloVeModel](src/main/java/com/github/chen0040/embeddings/GloVeModel.java) to create GloVe word embedding of different
+dimensions (e.g., 50, 100, 200, 300) 
+
+```java
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.github.chen0040.embeddings.GloVeModel;
+
+public class GloVeModelDemo {
+
+    private static final Logger logger = LoggerFactory.getLogger(GloVeModelDemo.class);
+
+    public static void main(String[] args) {
+        GloVeModel model = new GloVeModel();
+        model.load100();
+
+        logger.info("word2em size: {}", model.size());
+        logger.info("word2em dimension for individual word: {}", model.getWordVecDimension());
+
+        logger.info("father: {}", model.encodeWord("father"));
+        logger.info("mother: {}", model.encodeWord("mother"));
+        logger.info("man: {}", model.encodeWord("man"));
+        logger.info("woman: {}", model.encodeWord("woman"));
+        logger.info("boy: {}", model.encodeWord("boy"));
+        logger.info("girl: {}", model.encodeWord("girl"));
+        
+        logger.info("distance between boy and girl: {}", model.distance("boy", "girl"));
+
+
+        String doc = "The Zen of Python. Beautiful is better than ugly. Explicit is better than implicit. Simple is better than complex. Complex is better than complicated. Flat is better than nested. Sparse is better than dense. Readability counts. Special cases aren't special enough to break the rules.";
+
+        logger.info("doc: {}", model.encodeDocument(doc));
+
+
+    }
+```

src/main/java/com/github/chen0040/embeddings/GloVeModel.java

@@ -55,21 +55,59 @@ public float[] encodeWord(String word) {
         return null;
     }
 
+    public float[] encodeDocument(String sentence) {
+        sentence = filter(sentence);
+        String[] words = sentence.split(" ");
+
+        float[] vec = new float[dimension];
+        for(String word: words) {
+            String w = word.trim();
+            if(w.equals("")){
+                continue;
+            }
+            float[] word2vec = encodeWord(w);
+            if(word2vec == null) continue;
+            for(int i=0; i < dimension; ++i){
+                vec[i] += word2vec[i];
+            }
+        }
+
+        return vec;
+
+    }
+
+    private String filter(String sent) {
+        sent = sent.toLowerCase();
+        String[] punctuations = new String[] {",", ".", ";", ":", "?", "!", "\"", "'"};
+        for(String punt : punctuations) {
+            sent = sent.replace(punt, " " + punt);
+        }
+        return sent;
+    }
+
     public int size() {
         return word2em.size();
     }
 
+    public int getWordVecDimension() {
+        return dimension;
+    }
+
     public Map<String, float[]> load(String dirPath, int dimension){
         this.dimension = -1;
         word2em.clear();
         String sourceFile100 = getGloVeTextFileName(dimension);
         String filePath = dirPath + "/" + sourceFile100;
         File file = new File(filePath);
         if(!file.exists()){
+
             String zipFilePath = dirPath + "/glove.6B.zip";
             if(!new File(zipFilePath).exists()) {
+                logger.info("{} not found on local machine, downloading it from {}", zipFilePath, url);
                 if (!HttpClient.downloadFile(url, zipFilePath)) {
                     return word2em;
+                } else {
+                    logger.info("{} is downloaded", zipFilePath);
                 }
             }
 
@@ -78,6 +116,8 @@ public Map<String, float[]> load(String dirPath, int dimension){
             }
         }
 
+        logger.info("loading {} into word2em", filePath);
+
         try(BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(new File(filePath))))){
             String line;
             while((line=reader.readLine()) != null) {
@@ -103,6 +143,7 @@ public Map<String, float[]> load(String dirPath, int dimension){
     }
 
     private boolean unZip(String zipFilePath, String dirPath) {
+        logger.info("unzipping {} to {}", zipFilePath, dirPath);
         try {
             ZipFile zipFile = new ZipFile(zipFilePath);
             zipFile.extractAll(dirPath);
@@ -113,4 +154,23 @@ private boolean unZip(String zipFilePath, String dirPath) {
             return false;
         }
     }
+
+    public double distance(String word1, String word2) {
+        float[] vec1 = encodeWord(word1);
+        float[] vec2 = encodeWord(word2);
+
+        if(vec1 == null || vec2 == null) {
+            return -1f;
+        }
+
+        float result = 0;
+        for(int i=0; i < dimension; ++i) {
+            float v1 = vec1[i];
+            float v2 = vec2[i];
+
+            result += (v1 - v2) * (v1 - v2);
+        }
+
+        return Math.sqrt(result);
+    }
 }

src/main/java/com/github/chen0040/embeddings/GloVeModelDemo.java

@@ -1,10 +1,34 @@
 package com.github.chen0040.embeddings;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 public class GloVeModelDemo {
+
+    private static final Logger logger = LoggerFactory.getLogger(GloVeModelDemo.class);
+
     public static void main(String[] args) {
         GloVeModel model = new GloVeModel();
         model.load100();
-        System.out.println("word2em size: " + model.size());
+
+        logger.info("word2em size: {}", model.size());
+        logger.info("word2em dimension for individual word: {}", model.getWordVecDimension());
+
+        logger.info("father: {}", model.encodeWord("father"));
+        logger.info("mother: {}", model.encodeWord("mother"));
+        logger.info("man: {}", model.encodeWord("man"));
+        logger.info("woman: {}", model.encodeWord("woman"));
+        logger.info("boy: {}", model.encodeWord("boy"));
+        logger.info("girl: {}", model.encodeWord("girl"));
+
+        logger.info("distance between boy and girl: {}", model.distance("boy", "girl"));
+
+
+        String doc = "The Zen of Python. Beautiful is better than ugly. Explicit is better than implicit. Simple is better than complex. Complex is better than complicated. Flat is better than nested. Sparse is better than dense. Readability counts. Special cases aren't special enough to break the rules.";
+
+        logger.info("doc: {}", model.encodeDocument(doc));
+
+
     }
 
 }