package com.d2v.web.service;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.StringReader;
import java.net.URL;
import java.util.*;
import java.util.Map.Entry;
import java.nio.file.Paths;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.atomic.AtomicInteger;

import com.d2v.web.task.DocSimilarityMetaConfig;
import com.d2v.web.utils.SimpleDB;
import com.d2v.web.vo.*;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.deeplearning4j.models.embeddings.WeightLookupTable;
import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer;
import org.deeplearning4j.models.embeddings.wordvectors.WordVectors;
import org.deeplearning4j.models.word2vec.VocabWord;
import org.deeplearning4j.models.word2vec.Word2Vec;
import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
import org.deeplearning4j.text.sentenceiterator.BasicLineIterator;
import org.deeplearning4j.text.sentenceiterator.SentenceIterator;
import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.LowCasePreProcessor;
import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;
import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.ops.transforms.Transforms;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import com.google.gson.stream.JsonReader;

import info.debatty.java.stringsimilarity.Jaccard;

@Service
public class Doc2vecDetailService {

    private static final Logger log = LoggerFactory.getLogger(Doc2vecDetailService.class);

    public String extractMorpAnalyze(String morpJson) {
        String keyword = "";

        JsonReader reader = new JsonReader(new StringReader(morpJson));
        reader.setLenient(true);

        JsonArray morpArray = new JsonParser().parse(reader).getAsJsonArray();
        log.debug("morp : {}", morpArray);

        for (int i = 0; i < morpArray.size(); i++) {
            log.debug("morpArray : {}", morpArray.get(i));
            JsonObject wordObject = morpArray.get(i).getAsJsonObject();
            log.debug("wordObject : {}", wordObject);

            JsonPrimitive idxPri = wordObject.getAsJsonPrimitive("idx");
            log.debug("idxPri : {}", idxPri);
            int idxInt = idxPri.getAsInt();

            if (idxInt == -1) {
                keyword = keyword.replaceAll(";$", "");
                keyword += ";";
            } else {
                JsonPrimitive surfacePri = wordObject.getAsJsonPrimitive("surface");
                log.debug("surfacePri : {}", surfacePri);
                String surfaceStr = surfacePri.getAsString();

                JsonPrimitive tagPri = wordObject.getAsJsonPrimitive("tag");
                log.debug("tagPri : {}", tagPri);
                String tagStr = tagPri.getAsString();

                if (StringUtils.containsIgnoreCase(tagStr, "nn")) {
                    keyword += surfaceStr;
                    keyword += ";";
                }
                JsonElement wordElement = wordObject.get("contain");
                if (wordElement != null) {
                    if (!wordElement.isJsonNull()) {
                        JsonArray containWordArray = wordObject.getAsJsonArray("contain");
                        log.debug("containWordPri : {}", containWordArray);
                        if (containWordArray != null) {
                            for (int j = 0; j < containWordArray.size(); j++) {
                                JsonObject containWordObj = containWordArray.get(j).getAsJsonObject();

                                JsonPrimitive containSurfacePri = containWordObj.getAsJsonPrimitive("surface");
                                log.debug("containSurfacePri : {}", containSurfacePri);
                                String containSurfaceStr = containSurfacePri.getAsString();

                                JsonPrimitive containTagPri = containWordObj.getAsJsonPrimitive("tag");
                                log.debug("containTagPri : {}", containTagPri);
                                String containTagStr = containTagPri.getAsString();

                                if (StringUtils.containsIgnoreCase(containTagStr, "nn")) {
                                    keyword += containSurfaceStr;
                                    keyword += ";";
                                }
                            }
                        }
                    }
                }
            }
        }

        return keyword;
    }

    public String morpAnalyze(String morpJson) {
        String keyword = "";

        JsonReader reader = new JsonReader(new StringReader(morpJson));
        reader.setLenient(true);

        JsonArray morpArray = new JsonParser().parse(reader).getAsJsonArray();
        log.debug("morp : {}", morpArray);

        for (int i = 0; i < morpArray.size(); i++) {
            log.debug("morpArray : {}", morpArray.get(i));
            JsonObject wordObject = morpArray.get(i).getAsJsonObject();
            log.debug("wordObject : {}", wordObject);

            JsonPrimitive idxPri = wordObject.getAsJsonPrimitive("idx");
            log.debug("idxPri : {}", idxPri);
            int idxInt = idxPri.getAsInt();

            if (idxInt == -1) {
                keyword = keyword.replaceAll(";$", "");
                keyword += "\n";
            } else {
                JsonPrimitive surfacePri = wordObject.getAsJsonPrimitive("surface");
                log.debug("surfacePri : {}", surfacePri);
                String surfaceStr = surfacePri.getAsString();

                JsonPrimitive tagPri = wordObject.getAsJsonPrimitive("tag");
                log.debug("tagPri : {}", tagPri);
                String tagStr = tagPri.getAsString();

                if (StringUtils.containsIgnoreCase(tagStr, "nn")) {
                    keyword += surfaceStr;
                    keyword += ";";
                }
                JsonElement wordElement = wordObject.get("contain");
                if (wordElement != null) {
                    if (!wordElement.isJsonNull()) {
                        JsonArray containWordArray = wordObject.getAsJsonArray("contain");
                        log.debug("containWordPri : {}", containWordArray);
                        if (containWordArray != null) {
                            for (int j = 0; j < containWordArray.size(); j++) {
                                JsonObject containWordObj = containWordArray.get(j).getAsJsonObject();

                                JsonPrimitive containSurfacePri = containWordObj.getAsJsonPrimitive("surface");
                                log.debug("containSurfacePri : {}", containSurfacePri);
                                String containSurfaceStr = containSurfacePri.getAsString();

                                JsonPrimitive containTagPri = containWordObj.getAsJsonPrimitive("tag");
                                log.debug("containTagPri : {}", containTagPri);
                                String containTagStr = containTagPri.getAsString();

                                if (StringUtils.containsIgnoreCase(containTagStr, "nn")) {
                                    keyword += containSurfaceStr;
                                    keyword += ";";
                                }
                            }
                        }
                    }
                }
            }
        }

        return keyword;
    }

    public String selectLatestVersion(SimpleDB dbSet) {

        String version = "";

        List<Map<String, Object>> rsMap = dbSet.getDataFromSQL("select a.taskVersion from morp_result a group by a.taskVersion order by a.taskVersion desc limit 1");

        for (int i = 0; i < rsMap.size(); i++) {
            version = rsMap.get(i).get("taskVersion").toString();
        }

        return version;
    }

    public TaskDefineJson buildJsonObject(String extractJson) {

        TaskDefineJson taskJson = new TaskDefineJson();

        JsonObject usrTaskDefineObj = new JsonParser().parse(extractJson).getAsJsonObject();

        JsonObject choiceObj = usrTaskDefineObj.getAsJsonObject("choice");

        SelectTableVO tableVO = new SelectTableVO();

        JsonPrimitive sourceTablePri = choiceObj.getAsJsonPrimitive("sourceTableName");

        log.debug("Source Table JsonPri : {}", sourceTablePri);
        String sourceTableStr = sourceTablePri.getAsString();
        tableVO.setSourceTableName(sourceTableStr);

        JsonPrimitive sourceTableColumnPri = choiceObj.getAsJsonPrimitive("sourceTableColumn");

        log.debug("Source Table Column JsonPri : {}", sourceTableColumnPri);
        String sourceTableColumnStr = sourceTableColumnPri.getAsString();
        tableVO.setSourceTableColumn(sourceTableColumnStr);

        JsonPrimitive sourceTableKeywordColumnPri = choiceObj.getAsJsonPrimitive("sourceTableKeywordColumn");

        log.debug("Source Table Keyword Column JsonPri : {}", sourceTableColumnPri);
        String sourceTableKeywordColumnStr = sourceTableKeywordColumnPri.getAsString();
        tableVO.setSourceTableKeywordColumn(sourceTableKeywordColumnStr);

        JsonPrimitive targetTablePri = choiceObj.getAsJsonPrimitive("targetTableName");

        log.debug("Target Table JsonPri : {}", targetTablePri);
        String targetTableStr = targetTablePri.getAsString();
        tableVO.setTargetTableName(targetTableStr);

        JsonPrimitive targetTableColumnPri = choiceObj.getAsJsonPrimitive("targetTableColumn");

        log.debug("Target Table Column JsonPri : {}", targetTableColumnPri);
        String targetTableColumnStr = targetTableColumnPri.getAsString();
        tableVO.setTargetTableColumn(targetTableColumnStr);

        JsonPrimitive targetTableKeywordColumnPri = choiceObj.getAsJsonPrimitive("targetTableKeywordColumn");

        log.debug("Target Table Keyword Column JsonPri : {}", targetTableColumnPri);
        String targetTableKeywordColumnStr = targetTableKeywordColumnPri.getAsString();
        tableVO.setTargetTableKeywordColumn(targetTableKeywordColumnStr);

        taskJson.setSelectTable(tableVO);
        Doc2vecVO d2vVO = new Doc2vecVO();
        DocSettingVO docSettingVO = new DocSettingVO();

        JsonObject doc2vecObj = usrTaskDefineObj.getAsJsonObject("doc2vec");

        JsonObject docSettingObj = doc2vecObj.getAsJsonObject("docSetting");

        JsonPrimitive sourceDocIdPri = docSettingObj.getAsJsonPrimitive("sourceDocIdColumn");

        log.debug("Source Doc Id JsonPri : {}", sourceDocIdPri);
        String sourceDocIdStr = sourceDocIdPri.getAsString();
        docSettingVO.setSourceDocIdColumn(sourceDocIdStr);

        JsonPrimitive sourceDocNamePri = docSettingObj.getAsJsonPrimitive("sourceDocNameColumn");

        log.debug("Source Doc Name JsonPri : {}", sourceDocNamePri);
        String sourceDocNmStr = sourceDocNamePri.getAsString();
        docSettingVO.setSourceDocNameColumn(sourceDocNmStr);

        JsonPrimitive targetDocIdPri = docSettingObj.getAsJsonPrimitive("targetDocIdColumn");

        log.debug("Target Doc Id JsonPri : {}", targetDocIdPri);
        String targetDocIdStr = targetDocIdPri.getAsString();
        docSettingVO.setTargetDocIdColumn(targetDocIdStr);

        JsonPrimitive targetDocNamePri = docSettingObj.getAsJsonPrimitive("targetDocNameColumn");

        log.debug("Target Doc Name JsonPri : {}", targetDocNamePri);
        String targetDocNmStr = targetDocNamePri.getAsString();
        docSettingVO.setTargetDocNameColumn(targetDocNmStr);

        d2vVO.setDocSetting(docSettingVO);

        Doc2vecSettingVO doc2vecSettingVO = new Doc2vecSettingVO();

        JsonObject doc2vecSettingObj = doc2vecObj.getAsJsonObject("doc2vecSetting");

        JsonPrimitive iterationPri = doc2vecSettingObj.getAsJsonPrimitive("iteration");

        log.debug("Iteration JsonPri : {}", iterationPri);
        int iterationInt = iterationPri.getAsInt();
        doc2vecSettingVO.setIteration(iterationInt);

        JsonPrimitive batchSizePri = doc2vecSettingObj.getAsJsonPrimitive("batchSize");

        log.debug("batchSize JsonPri : {}", batchSizePri);
        int batchSizeInt = batchSizePri.getAsInt();
        doc2vecSettingVO.setBatchSize(batchSizeInt);

        JsonPrimitive epochsPri = doc2vecSettingObj.getAsJsonPrimitive("epochs");

        log.debug("epochs JsonPri : {}", epochsPri);
        int epochsInt = epochsPri.getAsInt();
        doc2vecSettingVO.setEpochs(epochsInt);

        JsonPrimitive layerSizePri = doc2vecSettingObj.getAsJsonPrimitive("layerSize");

        log.debug("layerSize JsonPri : {}", layerSizePri);
        int layerSizeInt = layerSizePri.getAsInt();
        doc2vecSettingVO.setLayerSize(layerSizeInt);

        JsonPrimitive windowSizePri = doc2vecSettingObj.getAsJsonPrimitive("windowSize");

        log.debug("windowSize JsonPri : {}", windowSizePri);
        int windowSizeInt = windowSizePri.getAsInt();
        doc2vecSettingVO.setWindowSize(windowSizeInt);

        JsonPrimitive minWordFrequencyPri = doc2vecSettingObj.getAsJsonPrimitive("minWordFrequency");

        log.debug("minWordFrequency JsonPri : {}", minWordFrequencyPri);
        int minWordFrequencyInt = minWordFrequencyPri.getAsInt();
        doc2vecSettingVO.setMinWordFrequency(minWordFrequencyInt);

        JsonPrimitive minLearningRatePri = doc2vecSettingObj.getAsJsonPrimitive("minLearningRate");

        log.debug("minLearningRate JsonPri : {}", minLearningRatePri);
        double minLearningRateDou = minLearningRatePri.getAsDouble();
        doc2vecSettingVO.setMinLearningRate(minLearningRateDou);

        JsonPrimitive learningRatePri = doc2vecSettingObj.getAsJsonPrimitive("learningRate");

        log.debug("learningRate JsonPri : {}", learningRatePri);
        double learningRateDou = learningRatePri.getAsDouble();
        doc2vecSettingVO.setLearningRate(learningRateDou);

        JsonPrimitive delimiterPri = doc2vecSettingObj.getAsJsonPrimitive("delimiter");

        log.debug("delimiter JsonPri : {}", delimiterPri);
        String delimiterStr = delimiterPri.getAsString();
        doc2vecSettingVO.setDelimiter(delimiterStr);

        d2vVO.setDoc2vecSetting(doc2vecSettingVO);

        SystemSettingVO systemSetVO = new SystemSettingVO();

        JsonObject systemSettingObj = doc2vecObj.getAsJsonObject("systemSetting");

        JsonPrimitive threadPartitionPri = systemSettingObj.getAsJsonPrimitive("threadPartition");

        log.debug("threadPartition JsonObj : {}", threadPartitionPri);
        int threadPartitionInt = threadPartitionPri.getAsInt();
        systemSetVO.setThreadPartition(threadPartitionInt);

        d2vVO.setSystemSetting(systemSetVO);
        taskJson.setDoc2vec(d2vVO);

        Doc2vecLogVO logVO = new Doc2vecLogVO();

        JsonObject doc2vecLogObj = usrTaskDefineObj.getAsJsonObject("doc2vecLog");

        JsonPrimitive inputFilePathPri = doc2vecLogObj.getAsJsonPrimitive("inputFilePath");

        log.debug("inputFilePath JsonObj : {}", inputFilePathPri);
        String inputFilePathStr = inputFilePathPri.getAsString();
        logVO.setInputFilePath(inputFilePathStr);

        JsonPrimitive inputFileNamePri = doc2vecLogObj.getAsJsonPrimitive("inputFileName");

        log.debug("inputFileName JsonObj : {}", inputFileNamePri);
        String inputFileNameStr = inputFileNamePri.getAsString();
        logVO.setInputFileName(inputFileNameStr);

        JsonPrimitive inputKeywordFilePathPri = doc2vecLogObj.getAsJsonPrimitive("inputKeywordFilePath");

        log.debug("inputKeywordFilePath JsonObj : {}", inputKeywordFilePathPri);
        String inputKeywordFilePathStr = inputKeywordFilePathPri.getAsString();
        logVO.setInputKeywordFilePath(inputKeywordFilePathStr);

        JsonPrimitive inputKeywordFileNamePri = doc2vecLogObj.getAsJsonPrimitive("inputKeywordFileName");

        log.debug("inputKeywordFileName JsonObj : {}", inputKeywordFileNamePri);
        String inputKeywordFileNameStr = inputKeywordFileNamePri.getAsString();
        logVO.setInputKeywordFileName(inputKeywordFileNameStr);

        JsonPrimitive outputFilePathPri = doc2vecLogObj.getAsJsonPrimitive("outputFilePath");

        log.debug("outputFilePath JsonObj : {}", outputFilePathPri);
        String outputFilePathStr = outputFilePathPri.getAsString();
        logVO.setOutputFilePath(outputFilePathStr);

        JsonPrimitive outputFileNamePri = doc2vecLogObj.getAsJsonPrimitive("outputFileName");

        log.debug("outputFileName JsonObj : {}", outputFileNamePri);
        String outputFileNameStr = outputFileNamePri.getAsString();
        logVO.setOutputFileName(outputFileNameStr);

        JsonPrimitive outputKeywordFilePathPri = doc2vecLogObj.getAsJsonPrimitive("outputKeywordFilePath");

        log.debug("outputKeywordFilePath JsonObj : {}", outputKeywordFilePathPri);
        String outputKeywordFilePathStr = outputKeywordFilePathPri.getAsString();
        logVO.setOutputKeywordFilePath(outputKeywordFilePathStr);

        JsonPrimitive outputKeywordFileNamePri = doc2vecLogObj.getAsJsonPrimitive("outputKeywordFileName");

        log.debug("outputKeywordFileName JsonObj : {}", outputKeywordFileNamePri);
        String outputKeywordFileNameStr = outputKeywordFileNamePri.getAsString();
        logVO.setOutputKeywordFileName(outputKeywordFileNameStr);

        taskJson.setDoc2vecLog(logVO);
        Doc2vecOutputVO outputVO = new Doc2vecOutputVO();

        JsonObject doc2vecOutputObj = usrTaskDefineObj.getAsJsonObject("doc2vecOuput");

        JsonPrimitive topDocSimiliarOuputTablePri = doc2vecOutputObj.getAsJsonPrimitive("topDocSimiliarOuputTable");

        log.debug("topDocSimiliarOuputTable JsonPri : {}", topDocSimiliarOuputTablePri);
        String topDocOutputTableStr = topDocSimiliarOuputTablePri.getAsString();
        outputVO.setTopDocSimiliarOuputTable(topDocOutputTableStr);

        JsonPrimitive topLimitExtractTypePri = doc2vecOutputObj.getAsJsonPrimitive("topLimitExtractType");

        log.debug("topLimitExtractType JsonPri : {}", topLimitExtractTypePri);
        String topLimitExtractTypeStr = topLimitExtractTypePri.getAsString();
        outputVO.setTopLimitExtractType(topLimitExtractTypeStr);

        JsonPrimitive topLimitExtractPri = doc2vecOutputObj.getAsJsonPrimitive("topLimitExtract");

        log.debug("topLimitExtract JsonPri : {}", topLimitExtractPri);
        int topLimitExtractInt = topLimitExtractPri.getAsInt();
        outputVO.setTopLimitExtract(topLimitExtractInt);

        JsonPrimitive lowDocSimiliarOutputTablePri = doc2vecOutputObj.getAsJsonPrimitive("lowDocSimiliarOutputTable");

        log.debug("lowDocSimiliarOutputTable JsonPri : {}", lowDocSimiliarOutputTablePri);
        String lowDocOutputTableStr = lowDocSimiliarOutputTablePri.getAsString();
        outputVO.setLowDocSimiliarOutputTable(lowDocOutputTableStr);

        JsonPrimitive lowLimitExtractTypePri = doc2vecOutputObj.getAsJsonPrimitive("lowLimitExtractType");

        log.debug("lowLimitExtractType JsonPri : {}", lowLimitExtractTypePri);
        String lowLimitExtractTypeStr = lowLimitExtractTypePri.getAsString();
        outputVO.setLowLimitExtractType(lowLimitExtractTypeStr);

        JsonPrimitive lowLimitExtractPri = doc2vecOutputObj.getAsJsonPrimitive("lowLimitExtract");

        log.debug("lowLimitExtract JsonPri : {}", lowLimitExtractPri);
        int lowLimitExtractInt = lowLimitExtractPri.getAsInt();
        outputVO.setLowLimitExtract(lowLimitExtractInt);

        taskJson.setDoc2vecOutput(outputVO);

        return taskJson;

    }

    public LoadingDocTable loadingSourceTable(SimpleDB dbset, DocSettingVO docSet, SelectTableVO choiceTable,
                                              LoadingDocTable loadTable, String latestVersion, Doc2vecTableVO doc2vecTableVO) {

        String sourceTableColumn = choiceTable.getSourceTableColumn();
        String sourceDomain = choiceTable.getSourceTableName();
        String sourceIdColumn = docSet.getSourceDocIdColumn();
        String sourceNameColumn = docSet.getSourceDocNameColumn();

        DocSimilarityMetaConfig.SOURCE_DOMAIN = sourceDomain;

        String[] spiltSourceTableColumn = sourceTableColumn.split(",");

        List<String> columnList = new ArrayList<String>(Arrays.asList(spiltSourceTableColumn));
        columnList.add(sourceIdColumn);
        columnList.add(sourceNameColumn);

        List<Map<String, Object>> maxVersion = dbset.getTableColumns(doc2vecTableVO.getItgtTable(), Arrays.asList(doc2vecTableVO.getItgtColVerLast()), "where " + doc2vecTableVO.getDomainCol() + "='" + sourceDomain + "' order by " + doc2vecTableVO.getItgtColVerLast() + " desc");
        DocSimilarityMetaConfig.procVer = Integer.valueOf(String.valueOf(maxVersion.get(0).get(doc2vecTableVO.getItgtColVerLast())));
        try {
            List<Map<String, Object>> prevVersion = dbset.getTableColumns(doc2vecTableVO.getD2vTable(), Arrays.asList(doc2vecTableVO.getD2vColVerLast()), "order by " + doc2vecTableVO.getD2vColVerLast() + " desc");
            DocSimilarityMetaConfig.prevVer = Integer.valueOf(String.valueOf(prevVersion.get(0).get(doc2vecTableVO.getD2vColVerLast())));
        }
        catch(Exception e) {
            DocSimilarityMetaConfig.prevVer = 0;
        }
        List<Map<String, Object>> dbResultList = dbset.getTableColumns(doc2vecTableVO.getItgtTable(), columnList, "where " + doc2vecTableVO.getDomainCol() + "='" + sourceDomain + "' and " + doc2vecTableVO.getItgtColVerLast() + " = '" + maxVersion.get(0).get(doc2vecTableVO.getItgtColVerLast()) + "'");

        log.info("Source Table Data Size : {}", dbResultList.size());

        String sourceText = "1";

        if (StringUtils.isNotEmpty(sourceText)) {
            loadTable.setSourceColumnList(columnList);
            loadTable.setSourceDbResultList(dbResultList);
            log.info("Source Table Data Complete...");
        } else {
            log.info("Source Table Data Fail...");
            System.exit(0);
        }

        return loadTable;

    }

    public LoadingDocTable loadingTargetTable(SimpleDB dbset, DocSettingVO docSet, SelectTableVO choiceTable,
                                              LoadingDocTable loadTable, String latestVersion, Doc2vecTableVO doc2vecTableVO) {

        String targetTableColumn = choiceTable.getTargetTableColumn();
        String targetDomain = choiceTable.getTargetTableName();
        String targetIdColumn = docSet.getTargetDocIdColumn();
        String targetNameColumn = docSet.getTargetDocNameColumn();

        String[] spiltTargetTableColumn = targetTableColumn.split(",");
        String[] targetDomainList = targetDomain.split(",");
        String targetDomainIn = "";
        int count = 0;
        for(String domain : targetDomainList) {
            count += 1;
            targetDomainIn += "'" + domain + "'";
            if(targetDomainList.length > count) {
                targetDomainIn += ",";
            }
        }

        List<String> columnList = new ArrayList<String>(Arrays.asList(spiltTargetTableColumn));
        columnList.add(targetIdColumn);
        columnList.add(targetNameColumn);

        List<Map<String, Object>> maxVersion = dbset.getTableColumns(doc2vecTableVO.getItgtTable(), Arrays.asList(doc2vecTableVO.getItgtColVerLast()), "order by " + doc2vecTableVO.getItgtColVerLast() + " desc");

        List<Map<String, Object>> dbResultList = dbset.getTableColumns(doc2vecTableVO.getItgtTable(), columnList, "where " + doc2vecTableVO.getDomainCol() + " in (" + targetDomainIn + ") and " + doc2vecTableVO.getItgtColVerLast() + " = '" + maxVersion.get(0).get(doc2vecTableVO.getItgtColVerLast()) + "'");
        log.info("Target Table Data Size : {}", dbResultList.size());

        String targetText = "1";

        if (StringUtils.isNotEmpty(targetText)) {
            loadTable.setTargetColumnList(columnList);
            loadTable.setTargetDbResultList(dbResultList);
            log.info("Target Table Data Complete...");
        } else {
            log.info("Target Table Data Fail...");
            System.exit(0);
        }

        return loadTable;

    }

    public String extractSourceWordTrainningKeyword(SimpleDB dbset, SelectTableVO choiceTable, String latestVersion) {
        String sourceTableNm = choiceTable.getSourceTableName();
        String sourceTableKeywordColumn = choiceTable.getSourceTableKeywordColumn();
        String[] spiltSourceTableKeywordColumn = sourceTableKeywordColumn.split(",");

        List<String> sourceTableKeywordColumnList = new ArrayList<String>(Arrays.asList(spiltSourceTableKeywordColumn));

        List<Map<String, Object>> dbResultList = dbset.getTableColumns(sourceTableNm, sourceTableKeywordColumnList, "where morp_task_version = '" + latestVersion + "'");

        String keywordTrainText = "";

        for (int i = 0; i < dbResultList.size(); i++) {
            for (int j = 0; j < sourceTableKeywordColumnList.size(); j++) {
                keywordTrainText += dbResultList.get(i).get(sourceTableKeywordColumnList.get(j)).toString();
                keywordTrainText += "\n";
            }
        }

        return keywordTrainText;
    }

    public String extractTargetWordTrainningKeyword(SimpleDB dbset, SelectTableVO choiceTable, String latestVersion) {
        String targetTableNm = choiceTable.getTargetTableName();
        String targetTableKeywordColumn = choiceTable.getTargetTableKeywordColumn();
        String[] spiltTargetTableKeywordColumn = targetTableKeywordColumn.split(",");

        List<String> targetTableKeywordColumnList = new ArrayList<String>(Arrays.asList(spiltTargetTableKeywordColumn));

        List<Map<String, Object>> dbResultList = dbset.getTableColumns(targetTableNm, targetTableKeywordColumnList, "where morp_task_version = '" + latestVersion + "'");

        String keywordTrainText = "";

        for (int i = 0; i < dbResultList.size(); i++) {
            for (int j = 0; j < targetTableKeywordColumnList.size(); j++) {
                keywordTrainText += dbResultList.get(i).get(targetTableKeywordColumnList.get(j)).toString();
                keywordTrainText += "\n";
            }
        }

        return keywordTrainText;
    }

    public WordVectors trainningText() {
        String filePath = "./input/vector.json";
        String jsonStr = "";
        int j;
        try {
            FileReader in = new FileReader(filePath);
            while ((j = in.read()) != -1) {
                char ch = (char) j;
                jsonStr += ch;
            }
        } catch (Exception e) {
            e.printStackTrace();
        }

        Gson gson = new Gson();
        W2vPropVO w2vPropObj = gson.fromJson(jsonStr, W2vPropVO.class);

        String fileNmPostfix = ".txt";
        String outputFilePath = w2vPropObj.getW2vLog().getOutputFilePath();
        String outputFileNm = w2vPropObj.getW2vLog().getOutputFileName();

        String outputFile = outputFilePath + outputFileNm + "_" + DocSimilarityMetaConfig.procVer +  fileNmPostfix;
//        String outputFile = "./test1.txt";

        WordVectors _wordVectors = null;

        try {
            File trainingFile = new File(outputFile);

            if (trainingFile.exists()) {
                log.info("Doc2vec Training DataSet Loading...");
                _wordVectors = WordVectorSerializer.readWord2VecModel(trainingFile);
                log.info("Doc2vec Training DataSet Load Complete...");
            }
        } catch (Exception e) {
            e.printStackTrace();
        }

        return _wordVectors;
    }

    public WordVectors trainningText(Word2vecVO w2vVo, File inputFile, String outputFile) {

        int batchSize = w2vVo.getBatchSize();
        int iteration = w2vVo.getIteration();
        int layerSize = w2vVo.getLayerSize();
        int windowSize = w2vVo.getWindowSize();
        int epochs = w2vVo.getEpochs();
        int minWordFrequency = w2vVo.getMinWordFrequency();
        double learningRate = w2vVo.getLearningRate();
        double minLearningRate = w2vVo.getMinLearningRate();

        WordVectors _wordVectors = null;
        SentenceIterator iter;

        try {
            iter = new BasicLineIterator(inputFile);
            TokenizerFactory t = new DefaultTokenizerFactory();
            t.setTokenPreProcessor(new LowCasePreProcessor());

            List<String> stopWord = new ArrayList<String>();

            Date startDate = new Date();
            long fileSize = inputFile.length();

            log.info("Word2vec Training Start...");

            Word2Vec w2v = new Word2Vec.Builder().batchSize(batchSize).allowParallelTokenization(true)
                    .stopWords(stopWord).minWordFrequency(minWordFrequency).iterations(iteration).layerSize(layerSize)
                    .learningRate(learningRate).seed(42).minLearningRate(minLearningRate).windowSize(windowSize)
                    .iterate(iter).epochs(epochs).tokenizerFactory(t).useAdaGrad(true).sampling(0.0001)
                    .negativeSample(2).build();

            w2v.fit();

            WordVectorSerializer.writeWord2VecModel(w2v, outputFile);

            _wordVectors = w2v;
            log.info("Word2vec Training Complete...");
            Date completeDate = new Date();
            log.info(String.valueOf(fileSize));
            log.info(String.valueOf(startDate));
            log.info(String.valueOf(completeDate));

        } catch (Exception e) {
            e.printStackTrace();
        }

        return _wordVectors;
    }

    public WordVectors trainningKeyword(String w2v, File w2vExistFile, File inputFile, String outputFile)
            throws FileNotFoundException {

        log.info("Load Existing Train Data...");
        Word2Vec word2Vec = WordVectorSerializer.readWord2VecModel(w2vExistFile);
        log.info("Loading Complete...");

        SentenceIterator iterator = new BasicLineIterator(inputFile);
        TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory();
        tokenizerFactory.setTokenPreProcessor(new LowCasePreProcessor());

        word2Vec.setTokenizerFactory(tokenizerFactory);
        word2Vec.setSentenceIterator(iterator);

        log.info("Doc2vec UpTraining Start...");
        word2Vec.fit();
        log.info("Doc2vec UpTraining Complete...");

        WordVectorSerializer.writeWord2VecModel(word2Vec, outputFile);

        WordVectors _wordVectors = word2Vec;

        return _wordVectors;

    }

    public WordVectors trainningKeyword(Doc2vecVO doc2vec, Doc2vecLogVO logVO) throws FileNotFoundException {

        String outputFilePath = logVO.getOutputFilePath();
        String outputFileName = logVO.getOutputFileName();
        String outputFile = outputFilePath + outputFileName;

        String inputKeywordFilePath = logVO.getInputKeywordFilePath();
        String inputKeywordFileName = logVO.getInputKeywordFileName();
        String inputKeywordFile = inputKeywordFilePath + inputKeywordFileName;
        File inputKeywordFileNm = new File(inputKeywordFile);

        String outputKeywordFilePath = logVO.getOutputKeywordFilePath();
        String outputKeywordFileName = logVO.getOutputKeywordFileName();
        String outputKeywordFile = outputKeywordFilePath + outputKeywordFileName;
        File outputKeywordFileNm = new File(outputKeywordFile);

        if (outputKeywordFileNm.exists()) {
            log.info("Load Existing Keyword Train Data...");
            Word2Vec word2Vec = WordVectorSerializer.readWord2VecModel(outputKeywordFileNm);
            log.info("Loading Existing Keyword Complete...");

            WordVectors _wordVectors = word2Vec;

            return _wordVectors;
        } else {
            log.info("Load Existing Train Data...");
            Word2Vec word2Vec = WordVectorSerializer.readWord2VecModel(outputFile);
            log.info("Loading  Complete...");

            SentenceIterator iterator = new BasicLineIterator(inputKeywordFileNm);
            TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory();
            tokenizerFactory.setTokenPreProcessor(new LowCasePreProcessor());

            word2Vec.setTokenizerFactory(tokenizerFactory);
            word2Vec.setSentenceIterator(iterator);

            log.info("Doc2vec UpTraining Start...");
            word2Vec.fit();
            log.info("Doc2vec UpTraining Complete...");

            WordVectorSerializer.writeWord2VecModel(word2Vec, outputKeywordFileNm);

            WordVectors _wordVectors = word2Vec;

            return _wordVectors;
        }

    }

    public Map<Integer, DocKeywordMapVO> extractingSourceDataKeyword(LoadingDocTable loadTable, DocSettingVO docSet,
                                                                     Map<Integer, DocKeywordMapVO> mappingFirstDomainKeyword) {

        List<String> sourceColumnList = loadTable.getSourceColumnList();
        List<Map<String, Object>> sourceDBMap = loadTable.getSourceDbResultList();
        String sourceDocIdCol = docSet.getSourceDocIdColumn();
        String sourceDocNmCol = docSet.getSourceDocNameColumn();

        for (int i = 0; i < sourceDBMap.size(); i++) {
            DocKeywordMapVO vo = new DocKeywordMapVO();
            String sourceDocId = "";
            String sourceDocNm = "";
            String sourceKeyword = "";
            for (int j = 0; j < sourceColumnList.size(); j++) {
                String columnStr = sourceColumnList.get(j);
                if (!StringUtils.equals(columnStr, sourceDocIdCol)) {
                    if (!StringUtils.equals(columnStr, sourceDocNmCol)) {
                        Object morpObj = sourceDBMap.get(i).get(columnStr);
                        String morpStr = transformObjectToString(morpObj);
//                        boolean judJson = $isJSONValid(morpStr);
//                        if(judJson){
//                            sourceKeyword += extractMorpAnalyze(morpStr);
//                        }else{
                        morpStr = morpStr.replaceAll("\n",";");
                        sourceKeyword += morpStr;
                        sourceKeyword += ";";
//                        }
                    }
                }

            }
            Object sourceDocObj = sourceDBMap.get(i).get(sourceDocIdCol);
            sourceDocId = transformObjectToString(sourceDocObj);
            Object sourceDocNmObj = sourceDBMap.get(i).get(sourceDocNmCol);
            sourceDocNm = transformObjectToString(sourceDocNmObj);

            sourceKeyword = sourceKeyword.replaceAll(" ", "_");
            sourceKeyword = sourceKeyword.replaceAll(";", "@");
            sourceKeyword = sourceKeyword.toLowerCase();

            List<String> docSimilarIdList = new ArrayList<String>();
            vo.setDocId(sourceDocId);
            vo.setDocIdList(docSimilarIdList);
            vo.setDocName(sourceDocNm);
            vo.setDocKeyword(sourceKeyword);

            int docHashCode = vo.hashCode();

            if (mappingFirstDomainKeyword.containsKey(docHashCode)) {
                DocKeywordMapVO docVO = mappingFirstDomainKeyword.get(docHashCode);
                List<String> existingDocIdList = docVO.getDocIdList();
                existingDocIdList.add(sourceDocId);
                docVO.setDocIdList(existingDocIdList);
                mappingFirstDomainKeyword.put(docHashCode, docVO);
            } else {
                mappingFirstDomainKeyword.put(docHashCode, vo);
            }
        }

        return mappingFirstDomainKeyword;

    }

    public Map<Integer, DocKeywordMapVO> extractingTargetDataKeyword(LoadingDocTable loadTable, DocSettingVO docSet,
                                                                     Map<Integer, DocKeywordMapVO> mappingSecondDomainKeyword) {

        List<String> targetColumnList = loadTable.getTargetColumnList();
        List<Map<String, Object>> targetDBMap = loadTable.getTargetDbResultList();
        String targetDocIdCol = docSet.getTargetDocIdColumn();
        String targetDocNmCol = docSet.getTargetDocNameColumn();

        Map<String, String> stopDocMap = new HashMap<String, String>();

//        for (int i = 0; i < stopDocRs.size(); i++) {
//            String stopDocId = stopDocRs.get(i).get("stop_doc_id").toString();
//            String stopDocNm = stopDocRs.get(i).get("stop_doc_nm").toString();
//            stopDocMap.put(stopDocId, stopDocNm);
//        }

        for (int i = 0; i < targetDBMap.size(); i++) {
            DocKeywordMapVO vo = new DocKeywordMapVO();
            String targetDocId = "";
            String targetDocNm = "";
            String targetKeyword = "";
            for (int j = 0; j < targetColumnList.size(); j++) {
                String columnStr = targetColumnList.get(j);
                if (!StringUtils.equals(columnStr, targetDocIdCol)) {
                    if (!StringUtils.equals(columnStr, targetDocNmCol)) {
                        Object morpObj = targetDBMap.get(i).get(targetColumnList.get(j));
                        String morpStr = transformObjectToString(morpObj);
                        boolean judJson = $isJSONValid(morpStr);
                        if(judJson){
                            targetKeyword += extractMorpAnalyze(morpStr);
                        }else{
                            morpStr = morpStr.replaceAll("\n",";");
                            targetKeyword += morpStr;
                            targetKeyword += ";";
                        }
                    }
                }
            }
            Object targetDocIdObj = targetDBMap.get(i).get(targetDocIdCol);
            targetDocId = transformObjectToString(targetDocIdObj);
            Object targetDocNmObj = targetDBMap.get(i).get(targetDocNmCol);
            targetDocNm = transformObjectToString(targetDocNmObj);

//            if(stopDocMap.containsKey(targetDocId))
//                continue;

            targetKeyword = targetKeyword.replaceAll(" ", "_");
            targetKeyword = targetKeyword.replaceAll(";", "@");
            targetKeyword = targetKeyword.toLowerCase();

            List<String> docSimilarIdList = new ArrayList<String>();
            vo.setDocId(targetDocId);
            vo.setDocIdList(docSimilarIdList);
            vo.setDocName(targetDocNm);
            vo.setDocKeyword(targetKeyword);

            int docHashCode = vo.hashCode();

            if (mappingSecondDomainKeyword.containsKey(docHashCode)) {
                DocKeywordMapVO docVO = mappingSecondDomainKeyword.get(docHashCode);
                List<String> existingDocIdList = docVO.getDocIdList();
                existingDocIdList.add(targetDocId);
                docVO.setDocIdList(existingDocIdList);
                mappingSecondDomainKeyword.put(docHashCode, docVO);
            } else {
                mappingSecondDomainKeyword.put(docHashCode, vo);
            }
        }

        return mappingSecondDomainKeyword;

    }

    public Map<Integer, DocKeywordListMapVO> extractSourceDocKeywordSpilt(
            Map<Integer, DocKeywordMapVO> mappingFirstDomainKeyword,
            Map<Integer, DocKeywordListMapVO> mappingFirstDomainKeywordList, String delimiter,
            Map<String,String> wordMap) {

        for (Entry<Integer, DocKeywordMapVO> e : mappingFirstDomainKeyword.entrySet()) {
            String[] splited =  e.getValue().getDocKeyword().split(delimiter);
            List<String> keywordList =  new ArrayList<>();

            for(String str :splited)
            {
                String word = wordMap.get(str);
                if(word!=null)
                {
                    keywordList.add(word);
                }
                else
                {
                    wordMap.put(str,str);
                    keywordList.add(str);
                }
            }

            //List<String> keywordList = new ArrayList<String>(Arrays.asList());
            //List<String> keywordList = new ArrayList<String>(keyword);
            DocKeywordListMapVO vo = new DocKeywordListMapVO();
            vo.setDocId(e.getValue().getDocId());
            vo.setDocName(e.getValue().getDocName());
            vo.setDocIdList(e.getValue().getDocIdList());
            vo.setDocKeywordList(keywordList);
            mappingFirstDomainKeywordList.put(e.getKey(), vo);
        }

        return mappingFirstDomainKeywordList;
    }

    public Map<Integer, DocKeywordListMapVO> extractTargetDocKeywordSpilt(
            Map<Integer, DocKeywordMapVO> mappingSecondDomainKeyword,
            Map<Integer, DocKeywordListMapVO> mappingSecondDomainKeywordList, String delimiter,
            Map<String,String> wordMap) {

        for (Entry<Integer, DocKeywordMapVO> e : mappingSecondDomainKeyword.entrySet()) {
            String[] splited =  e.getValue().getDocKeyword().split(delimiter);
            List<String> keywordList =  new ArrayList<>();

            for(String str :splited)
            {
                String word = wordMap.get(str);
                if(word!=null)
                {
                    keywordList.add(word);
                }
                else
                {
                    wordMap.put(str,str);
                    keywordList.add(str);
                }
            }


            DocKeywordListMapVO vo = new DocKeywordListMapVO();
            vo.setDocId(e.getValue().getDocId());
            vo.setDocName(e.getValue().getDocName());
            vo.setDocIdList(e.getValue().getDocIdList());
            vo.setDocKeywordList(keywordList);
            mappingSecondDomainKeywordList.put(e.getKey(), vo);
        }

        return mappingSecondDomainKeywordList;
    }

    public Map<Integer, TargetDocVecVO> $tempTargetDocSave(
            Map<Integer, DocKeywordListMapVO> mappingTargetDocKeywordList, VocabCache<VocabWord> vocab,
            WeightLookupTable<VocabWord> lookupTable, int layerSize) {

        //결과
        Map<Integer, TargetDocVecVO> targetDocVectorSave = new ConcurrentHashMap<Integer, TargetDocVecVO>();

        //문서 Hash랑 문서 및 키워드리스트에서 문서ID랑 객체를 각각에 대해서
        mappingTargetDocKeywordList.entrySet().parallelStream().forEach(targetDoc -> {

            String targetDocId = String.valueOf(targetDoc.getValue().getDocId());
            String targetDocSimilarIdStr = "";
            if (CollectionUtils.isNotEmpty(targetDoc.getValue().getDocIdList())) {
                Set<String> targetDocIdSet = new LinkedHashSet<String>(targetDoc.getValue().getDocIdList());
//                for (int i = 0; i < targetDoc.getValue().getDocIdList().size(); i++) {
//                    targetDocIdSet.add(targetDoc.getValue().getDocIdList().get(i));
//                }
                targetDocSimilarIdStr = targetDocIdSet.toString();
            }
            String targetDocNm = String.valueOf(targetDoc.getValue().getDocName());

            List<String> secondDomainKeywordList = targetDoc.getValue().getDocKeywordList();
            //layerSize: vectorSize
            INDArray SecondDocVector = Nd4j.create(layerSize);

            for (int i = 0; i < secondDomainKeywordList.size(); i++) {
                INDArray array = $extractArrayValue(lookupTable, vocab, secondDomainKeywordList.get(i));
                if (array != null) {
                    SecondDocVector = SecondDocVector.add(array);
                }
            }

            SecondDocVector = SecondDocVector.div(secondDomainKeywordList.size());
            TargetDocVecVO vo = new TargetDocVecVO();
            vo.setDocId(targetDocId);
            vo.setDocIdList(targetDocSimilarIdStr);
            vo.setDocNm(targetDocNm);
            vo.setDocVector(SecondDocVector);
            targetDocVectorSave.put(targetDoc.getKey(), vo);
        });

        return targetDocVectorSave;

    }

    private INDArray $extractArrayValue(WeightLookupTable<VocabWord> lookupTable, VocabCache<VocabWord> vocab,
                                        String word) {
        if (vocab.containsWord(word)) {
            INDArray wordArray = lookupTable.vector(word);
            return wordArray;
        } else {
            return null;
        }
    }

    private double $extractDocSimilarity(INDArray sourceDocVec, INDArray targetDocVec) {
        double similarity = 0.0;
        similarity = Math.abs(Transforms.cosineSim(sourceDocVec, targetDocVec));
        return similarity;
    }

    private String transformObjectToString(Object obj) {
        String targetNm = "";

        if (obj != null) {
            targetNm = obj.toString();
            targetNm = targetNm.trim();
        }

        return targetNm;
    }

    private boolean $isJSONValid(String test) {
        try {
            new JSONObject(test);
        } catch (JSONException ex) {
            try {
                new JSONArray(test);
            } catch (JSONException ex1) {
                return false;
            }
        }
        return true;
    }
}
