/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package testmeetup;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;

/**
 * This class makes conversions between the differents types of datasets we use in this program.
 * The list of formats used in the program is the following:
 *      1. Raw Events File
 *      2. Raw RSPV File
 *      3. Raw Member File
 *      4. RSVP Filtered File
 *      5. RSVP Datafiles
 *      6. Member Datafiles
 *      7. Splitted RSVP Datafiles    
 * 
 *  - Raw files are responses from the Meetup API
 *  - RSVP Filtered is the same format, but less data because it was filtered with a condition (usually distance)
 *  - Datafiles are processed data, with a special format based in csv (comma separated values) made specially to work with librec.
 *  - Splitted RSVP datafiles are the same but separated by number of RSVPS to make tests.
 * @author Oriol
 */
public class FileConverter {
    
    /**
     * Converts multiple Raw RSVP Files to a RSVP Datafile and a Member Datafile. Reads the JSON of the Raw file, gets the important information (MemberID, EventID, Rating, [optional] Timestamp)
     * and saves it to RSVP Datafile. Member Datafile only has memberID and new Member ID (The new ID we're giving to users).
     * The use of another ID is to be consistent with the type of ID, given that MeetupAPI sometimes returns long or Strings. The new ID is only a auto-incremental int.
     * It also can generate two extra files, member and item files. Even though it overloads this function a bit, I did this also here I did not trust myself to read the file again in another function
     * and not mess up the translation of old ID -> new ID. This way the translation of new ID is centralized and all files using the new ID are generated here also.
     * 
     * @param fileNames Files to convert to RSVP Datafile.
     * @param uirt If true, the format of the converted file will be UIRT (User - Item - Rating - Timestamp). If false, only UIR (User - Item - Rating)
     * @param extraFiles if true, it will generate extra files containing the OLD ID, NEW ID of user and items. 
     */
    public static void RSVPtoTXT(ArrayList<String> fileNames,ArrayList<String> fileNamesEvents, boolean uirt, boolean extraFiles){
        StringBuilder TXTData = new StringBuilder();  
        StringBuilder memberData = new StringBuilder();
        StringBuilder eventData = new StringBuilder();
        
        int newIDUser = 1;
        int newIDEvent = 1;
        JSONArray jsonFile;
        JSONObject line, event,member;
        String memberID, eventID;
        Long timestamp;
        Integer ret,count;
        
        // Maps will be used to keep track of different values between files: the same user and item Id can appear on different files, so we are going to keep
        // a Map while parsing all the files.
        Map<String, Integer> hmUsers =      new HashMap<String, Integer>();
        Map<String, Integer> hmEvents =     new HashMap<String, Integer>();
        Map<String, Integer> hmUserCount =  new HashMap<String, Integer>();
        Map<String, Integer> hmEventCount = new HashMap<String, Integer>();
        Map<String, Long> hmEventTimes =    new HashMap<String, Long>();
        
        //IF UIRT is true, we need to get timestamp data, which is located in the Event file. This is the only information we will get from this file
        if(uirt){
            for(int f = 0 ; f < fileNamesEvents.size(); f++){
                jsonFile = FileUtils.loadData(fileNamesEvents.get(f));
                for (int i = 0; i < jsonFile.size(); i++) {
                    line = (JSONObject) jsonFile.get(i);
                    timestamp = (Long) line.get("time");
                    eventID = (String) line.get("id");
                    if(hmEventTimes.get(eventID) == null){  //we get Time and (old) ID of the event and save each unique ID with its time
                        hmEventTimes.put(eventID, timestamp);
                    }
                }
            }
        }
        
        for(int f = 0 ; f < fileNames.size(); f++){   
            jsonFile = FileUtils.loadData(fileNames.get(f));


            for (int i = 0; i < jsonFile.size(); i++) {
                line = (JSONObject) jsonFile.get(i);
                event =  (JSONObject) line.get("event");
                member = (JSONObject) line.get("member");
                
                if( event == null || member == null)
                    continue;
                
                memberID = member.get("id").toString();
                eventID = event.get("id").toString();
                if(memberID == null || eventID == null){
                    System.out.println("ERROR IN FILECONVERTER: an ID was null. Full trace:   " +  line.toString());
                    continue;
                }

                //Add the info for one line: USER_ID,ITEM_ID,RATING (+1,0,-1)
                
                //User ID translation
                ret = hmUsers.putIfAbsent(memberID, newIDUser);
                if(ret != null){ //PutIfAbsent found memberId, doesnt create entry and returns id
                    TXTData.append(ret).append(",");
                }else{ //PutIfAbsent didnt find memberId, creates entry; newIDUser is the new id
                    TXTData.append(newIDUser).append(",");
                    newIDUser++;
                }
                
                if(extraFiles){
                    //Save event ID to later add position (operation 7)
                    count = hmUserCount.get(memberID);
                    if(count == null){
                        hmUserCount.put(memberID, 1);
                    }else{
                        hmUserCount.put(memberID, count+1);
                        if(count == 1){
                            //Create line. For now it only has realID,newID. Later we have to add lat,lon
                            memberData.append(memberID).append(",").append(hmUsers.get(memberID)).append("\n");
                        }
                    }
                }
                
                //event ID translation
                ret = hmEvents.putIfAbsent(eventID, newIDEvent);
                if(ret != null){ //PutIfAbsent found memberId, doesnt create entry and returns id
                    TXTData.append(ret).append(",");
                }else{ //PutIfAbsent didnt find memberId, creates entry; newIDUser is the new id
                    TXTData.append(newIDEvent).append(",");
                    newIDEvent++;
                }

                if(extraFiles){
                    //Save event ID to later add position (operation 7)
                    count = hmEventCount.get(eventID);
                    if(count == null){
                        hmEventCount.put(eventID, 1);
                    }else{
                        hmEventCount.put(eventID, count+1);
                        if(count == 1){
                            //Create line. For now it only has realID,newID. Later we have to add lat,lon
                            eventData.append(eventID).append(",").append(hmEvents.get(eventID)).append("\n");
                        }
                    }
                }
                
                // RSVP rating transformation into {-1,1}
                if(line.get("response").equals("no")){
                    TXTData.append("-1");
                }else{ //Other cases: yes, waitlist (yes but no places left on event), yes_pending_payment (yes but need to confirm payment for the event)...
                    TXTData.append("1");
                }
                if(uirt){
                    timestamp = hmEventTimes.get(eventID);
                    if(timestamp == null){
                        System.out.println("ERROR IN FILECONVERTER: a TIMESTAMP was null" +  line.toString());
                        continue;
                    }
                    TXTData.append(",").append(timestamp);
                }
                
                TXTData.append("\n");
            }
        }
        
        //Save information
        String newFile = FileUtils.getFolder(fileNames.get(0)) + "rsvpDF" + fileNames.size() + ".txt";
        FileUtils.saveData(TXTData.toString(), newFile);
        if(memberData.length() != 0){
            newFile = FileUtils.getFolder(fileNames.get(0)) + "membersDF -" +  fileNames.size() + ".txt";
            FileUtils.saveData(memberData.toString(), newFile);
        }
        if(eventData.length() != 0){
            newFile = FileUtils.getFolder(fileNames.get(0)) + "itemsDF -" +  fileNames.size() + ".txt";
            FileUtils.saveData(eventData.toString(), newFile);
        }

    }
    
    

     
    //Load Events datafiles, readlines, filter by distance, save id in array
    /**
     * Converts a RSVP Datafile to a RSVP Filtered Datafile.
     * @param fileNames RSPVP Datafiles to filter.
     * @param maxDistance Distance to filter.
     * @return 
     */
    public static ArrayList<String> filterEventsByDistance(ArrayList<String> fileNames, double maxDistance){
        ArrayList<String> filteredEvents = new ArrayList<>();
        for(int f = 0 ; f < fileNames.size(); f++){
            JSONArray jsonFile = FileUtils.loadData(fileNames.get(f));
            JSONObject line;
            double distance;
            for(int i=0; i<jsonFile.size(); i++){
                line = (JSONObject) jsonFile.get(i);
                distance = Double.parseDouble(line.get("distance").toString());
                if(distance <= maxDistance){ //CONDITION
                    filteredEvents.add(line.get("id").toString());
                }
            }
        }
        return filteredEvents;
    }   
    
    /**
     * Converts a RSVP Dataset into multiple RSVP Datasets separated by number of RSVPS. The frequencies and number of outputs are fixed. 
     * The algorithm sorts the files by time of the event
     * @param fileName Name of the RSVP Dataset
     */
    public static void splitByNRSVP(String fileName){
        //Read the datafile as txt
        ArrayList<ArrayList<String>> files = new ArrayList<ArrayList<String>>();
        for (int split = 0; split < 7; split++) {
            files.add(new ArrayList<String>()); //Initialize arraylists inside the main arraylist
        }
        try {
            List<String> lines = FileUtils.loadDataTxt(fileName);
            Collections.sort(lines);
            String userID;
            ArrayList<String> fullUser = new ArrayList<String>();
            int pos;
            
            userID = lines.get(0).split(",")[0];
            if( userID.equals("") ){ //First line bug
                lines.remove(0);
                userID = lines.get(0).split(",")[0];
            }
            for (int i = 1; i < lines.size(); i++) {
                if(userID.equals(lines.get(i).split(",")[0])){ //its the same user again; up the counter and info
                    fullUser.add(lines.get(i));
                }else{ //Diferent user; save the information in corresponding file and restart count and info
                    int count = fullUser.size();
                    if(count <= 1){ //1
                        pos = 0;
                    } else if( count == 2){ //2
                        pos = 1;
                    } else if( count == 3){ //3
                        pos = 2;
                    }else if (count == 4 || count == 5){ //4-5
                        pos = 3;
                    }else if(count > 5 && count <= 10){ //6-10
                        pos = 4;
                    }else if(count > 10 && count <= 20){ //11-20
                        pos = 5;
                    }else{  //>20
                        pos = 6;
                    }
                    for (int j = 0; j < fullUser.size(); j++) {
                        files.get(pos).add(fullUser.get(j));
                    }
                    //Next user
                    userID = lines.get(i).split(",")[0];
                    fullUser = new ArrayList<String>();
                    fullUser.add(lines.get(i));
                    count = 1;  //count and info = 1 because we're counting the first user
                }
            }
            for(int i = 0; i< files.size(); i++){
                files.get(i).sort(new Comparator<Object>() { //SORTING BY TIME
                    public int compare(Object o1, Object o2) {
                    
                        String s1 = (String) o1;
                        String s2 = (String) o2;
                        
                        String time1 = s1.split(",")[3];
                        String time2 = s2.split(",")[3];
                        //System.out.println("[" +time1.substring(0,time1.length()-1)  + "] , [" + time2.substring(0,time2.length()-1) + "]");
                        Long long1 = Long.parseLong(time1.substring(0,time1.length()-2));
                        Long long2 = Long.parseLong(time2.substring(0,time2.length()-2));

                        if(long1 > long2){
                            return 1;
                        }else if(long2 > long1){
                            return -1;
                        }
                        return 0;
                    }
                });
            }
            for(int i = 0; i< files.size(); i++){
                String newFileName = FileUtils.getFolder(fileName) + "splittedDF"  + i + ".txt";
                String stringFile = "";
                for (int line = 0; line < files.get(i).size(); line++) {
                    stringFile += files.get(i).get(line) + "\n";
                }
                FileUtils.saveData(stringFile, newFileName);
            }
        
        } catch (Exception ex) {
            Logger.getLogger(FileConverter.class.getName()).log(Level.SEVERE, null, ex);
        }
        
    }

    
    
    /**
     * Creates a RSVP Filtered based on a Event Filtered.
     * @param fileNames
     * @param eventIds 
     */
    public static void filterRSVPByEventIds(ArrayList<String> fileNames, ArrayList<String> eventIds){

        for(int f = 0 ; f < fileNames.size(); f++){
            JSONArray jsonFile = FileUtils.loadData(fileNames.get(f));
            JSONObject line,event;
            int i=0;
            while(i<jsonFile.size()){
                line = (JSONObject) jsonFile.get(i);
                event = (JSONObject) line.get("event");
                if(event != null && !eventIds.contains((String)event.get("id"))){ //CONDITION
                    jsonFile.remove(i);
                }else{
                    i++;
                }
            }
            String newFile = FileUtils.getFolder(fileNames.get(0)) + "FilteredRSVP" + (f+1)  + ".txt";
            FileUtils.saveData(jsonFile.toString() , newFile);
        }
    }   

    /**
     * Method that completes a file containing item ID produced in operation 4 with location information. 
     * To do it, it also loads event files, where the location information of each event is present.
     * @param eventFiles Auxiliar files to search the location position of each event
     * @param itemFile file with ID, which will be completed with event positions
     */
    public static void addPosEvents(ArrayList<String> eventFiles,String itemFile){
        List<String> lines = FileUtils.loadDataTxt(itemFile);
        List<String> seenEvents = new ArrayList<>();
        StringBuilder itemData = new StringBuilder();
        String itemID;
        JSONArray jsonFile;
        JSONObject linejson,event,venue;
        Double lat,lon;
        int l = 0;
        boolean found;
        
        for (int f = 0; f < eventFiles.size(); f++) {
            jsonFile = FileUtils.loadData(eventFiles.get(f));
            for (int i = 0; i < jsonFile.size(); i++) {
                linejson = (JSONObject) jsonFile.get(i);
                event =  (JSONObject) linejson.get("event");
                itemID =  (String) event.get("id");
                if(seenEvents.indexOf(itemID)==-1){ 
                    seenEvents.add(itemID);
                }else{
                    continue;
                }
                l = 0;
                found = false;
                while(!found && l<lines.size()){
                    String line = lines.get(l);
                    if(itemID.equals(line.split(",")[0])){
                        venue =  (JSONObject) linejson.get("venue"); //Lat and Lon coordinates appear inside the venue tag, which is not always present (i think around 2/3 of the time is present)
                        if(venue != null){
                            lat = (Double) venue.get("lat");
                            lon = (Double) venue.get("lon");
                            itemData.append(line).append("," + lat + "," + lon + "\n");
                        }
                        found = true;
                    }
                    l++;
                }
            }
        }
        
        String newFile = FileUtils.getFolder(itemFile) + "itemsDFComplete.txt";
        FileUtils.saveData(itemData.toString(), newFile);
    }
    

    /**
     * Loads file of member IDs produced in operation 4. In order to complete this file with position information, for every user ID it needs to make a query, wait and process the result.
     * This operation might take a lot of time.
     * @param fileName
     * @param folder 
     */
    public static void getLatLon(String fileName,String folder){
        String line;
        String lat,lon;
        JSONArray response = null;
        String memberID;
        QueryMember qmember = new QueryMember();
        StringBuilder memberData = new StringBuilder();
        List<String> members = FileUtils.loadDataTxt(fileName);
        for (int i = 0; i < members.size(); i++) {
            line = members.get(i);
            memberID = line.split(",")[0];
            qmember.setMemberID(memberID);
            try {
                response = getMember(qmember);
                JSONObject item = (JSONObject) response.get(0);
                lat = (String) item.get("lat").toString();
                lon = (String) item.get("lon").toString();
                memberData.append(line).append(",").append(lat).append(",").append(lon).append("\n");
                
            } catch (Exception ex) {
                Logger.getLogger(MeetupDataManager.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
        FileUtils.saveData(memberData.toString(),folder  +"\\MembersLatLon.txt");
    }
    
    /**
     * Given a Member Query, it sends and processes the result into a JSONArray.
     * @param qmember Query object
     * @return response from the query
     * @throws Exception error from connection
     */
    public static JSONArray getMember(QueryMember qmember) throws Exception{
        JSONParser parser = new JSONParser();
        JSONArray response = new JSONArray();
        JSONObject item = null;
        qmember.generateQuery();
        item = (JSONObject) parser.parse(QuerySender.sendQuery(qmember.getURL()));
        response.add(item);
        return response;
    }
   
    /**
     * Creates all the event files into a lat,lon, distance file. This file purpose is to be plotted using external tools.
     * @param eventFiles 
     */
    public static void EventsToMap(ArrayList<String> eventFiles) {
        JSONArray jsonFile1;
        JSONObject line,venue;
        Double lat,lon,distance;
        StringBuilder mapData = new StringBuilder();
        for (int f1 = 0; f1 < eventFiles.size(); f1++) {
            jsonFile1 = FileUtils.loadData(eventFiles.get(f1));
            for (int i1 = 0; i1 < jsonFile1.size(); i1++) {
                line = (JSONObject) jsonFile1.get(i1);
                venue =  (JSONObject) line.get("venue");
                distance = 0.0;
                try{
                    distance = (Double) line.get("distance");
                }catch(Exception ex){
                    // Sometimes an exception will be thrown even though distance is supossed to be always double. 
                    //In some cases, distance is exactly 0 (parsed as Long for some reason), and it jumps here. Distance is 0 anyway, so nothing to do here.
                }
                
                if(venue != null){
                    try{
                        lat = (Double) venue.get("lat");
                        lon = (Double) venue.get("lon");
                        if(distance <= 25){ //1 = inside
                            mapData.append(lat + "," + lon + "," + 1 + "\n");        
                        }else{ //0 = outside
                            mapData.append(lat + "," + lon + "," + 0 + "\n");        
                        }
                    }catch(Exception ex){
                    }
                }
            }
        }
        String newFile = FileUtils.getFolder(eventFiles.get(0)) + "MapEventCoords.txt";
        FileUtils.saveData(mapData.toString(), newFile);
    }
    
    
    /**
     * Same as Events to Map, this function pretends to create a file to be used by external tools to plot. 
     * When using this function, please make sure the timezone is correct for the city, or pass it as parameter.
     * @param filename 
     */
    public static void TimestampsToDate(String filename){
        Calendar calendarobject = Calendar.getInstance();
        calendarobject.setTimeZone(TimeZone.getTimeZone("GMT-8:00"));  //Hard coded; please change depending on city
        int day,hour,rating;
        List<String> lines = FileUtils.loadDataTxt(filename);
        String line;
        Long timestamp;
        StringBuilder timeData = new StringBuilder();
        
        for (int i = 1; i < lines.size(); i++) {
            line = lines.get(i);
            if(line.split(",").length > 3){
                timestamp = Long.parseLong(line.split(",")[3]);
                rating = Integer.parseInt(line.split(",")[2]);
                day = getDayOfTheWeek(calendarobject,timestamp);
                hour = getHour(calendarobject,timestamp);
                
                
                
                timeData.append(day + "," + hour + "," + rating + "\n");
            }
        }
        String newFile = FileUtils.getFolder(filename) + "TimeData.txt";
        FileUtils.saveData(timeData.toString(), newFile);
    }
    
    //Auxiliar functions to get day and hour
    public static int getDayOfTheWeek(Calendar calendarobject,long time){
        Date dateobject = new Date(time);
        calendarobject.setTime(dateobject);
        return calendarobject.get(Calendar.DAY_OF_WEEK);
    }
        
    public static int getHour(Calendar calendarobject,long time){
        Date dateobject = new Date(time);
        calendarobject.setTime(dateobject);
        return calendarobject.get(Calendar.HOUR_OF_DAY);
    }
    
    
    
}
    
  



