---==================================== --Program.cs - Start ---==================================== using System; using System.Collections.Generic; using System.Configuration; using System.IO; using System.Threading.Tasks; namespace CDXFileUploadService { class Program { static void Main(string[] args) { try { string fileName = ""; string qStr = ""; string excelFileRecordStr = ""; string CDXFileLastUploadDate = ""; DateTime processBeginDate = DateTime.Now; string procesSts = ""; bool fileUploadStatus = false; string fPath = ConfigurationManager.AppSettings["filePath"]; string cdxUploadFileName = ""; string queryString = ""; // string fileName = "ActivityDataFile.xlsx"; // string queryString = "importConfigurationId=8121&newOrExistingData=0&uponCompletion=2&uponCompletionCondition=2&fileType=XLSX&worksheetsToImport=1&ignoreFirstRowOfFile=True&fileId="; // var arguments = new Dictionary(); // args = new string[] { "CDXGetActivityFileData", " "}; //args[0] = "CDXGetActivityFileData"; //args[1] = " "; for (int i = 0; i < args.Length; i++) { Console.WriteLine($"Arg[{i}] = [{args[i]}]"); Console.WriteLine("args length: " + args.Length); } if (args != null && args.Length == 2) { fileName = args[0]; qStr = args[1]; if (fileName != "CDXGetActivityFileData") { cdxUploadFileName = fPath + fileName; } } queryString = qStr; //test // Projects file qstr //fileName = "ProjectsDataFile.xlsx"; //qStr = "importConfigurationId=7174&newOrExistingData=0&uponCompletion=2&uponCompletionCondition=2&fileType=XLSX&worksheetsToImport=1&ignoreFirstRowOfFile=True&fileId="; //Stations file qstr // fileName = "StationsDataFile.xlsx"; // qStr = "importConfigurationId=7384&newOrExistingData=0&uponCompletion=2&uponCompletionCondition=2&fileType=XLSX&worksheetsToImport=1&ignoreFirstRowOfFile=True&fileId="; //Activity file qstr //fileName = "ActivityDataFile.xlsx"; //qStr = "importConfigurationId=8121&newOrExistingData=0&uponCompletion=2&uponCompletionCondition=2&fileType=XLSX&worksheetsToImport=1&ignoreFirstRowOfFile=True&fileId="; //queryString = qStr; //cdxUploadFileName = fPath + fileName; //Create CDXGetActivityFileData // fileName = "CDXGetActivityFileData"; //queryString = qStr; //test Logger.WriteLog(" "); Logger.WriteLog("================New Log Process Start================"); Logger.WriteLog("EWQX-EPA Web Service process started at: " + DateTime.Now); DataActions GetFileData = new DataActions(); //================================================================================== // Execute stored procedure and create Projects, Stations and Activity Excel files //================================================================================== Logger.WriteLog("Execute stored procedure to create Projects, Stations and Activity Excel files"); if (fileName == "ProjectsDataFile.xlsx") { excelFileRecordStr = GetFileData.getStoredProcData("CDXFileUploadProjectsData", "", processBeginDate); } else if (fileName == "StationsDataFile.xlsx") { excelFileRecordStr = GetFileData.getStoredProcData("CDXFileUploadStationsData", "", processBeginDate); } else if (fileName == "ActivityDataFile.xlsx") { //excelFileRecordStr = GetFileData.getStoredProcData("CDXFileUploadActivityData", "", processBeginDate); excelFileRecordStr = "1"; } else if (fileName == "CDXGetActivityFileData") { excelFileRecordStr = GetFileData.getActivityData("CDXGetActivityFileData", "", processBeginDate); //========================================================================================= //Insert a record with CDXFileUpload process status into database ProcessStatusDetail table // Successfully created CDX Activity file data and is ready to extract from database and create ActivityDataFile.xlsx //========================================================================================= procesSts = "CDXFileUploadSuccess-Created Activity data with status: " + excelFileRecordStr; // procesSts = "CDXFileUploadSuccess-Created Activity data status: "; //CDXFileLastUploadDate = GetFileData.getStoredProcData("CDXFileUploadStatus", procesSts, processBeginDate); if (excelFileRecordStr == "LabParmsAndActivityDataAnalyticsSuccessful") { excelFileRecordStr = GetFileData.getStoredProcData("CDXFileUploadActivityData", "", processBeginDate); } excelFileRecordStr = "2"; } //================================================================================== //Upload files into CDX-EPA web site //================================================================================== // Upload Projects file to CDX-EPA site // if (excelFileRecordStr == "1" || fileName == "ActivityDataFile.xlsx") if (excelFileRecordStr == "1") { Logger.WriteLog("Upload files into CDX-EPA web site"); fileUploadStatus = GetFileData.CDXfileUploadProcessAsync(fileName,cdxUploadFileName, queryString).GetAwaiter().GetResult(); } if (excelFileRecordStr == "2") { Logger.WriteLog("Compiled Activity data is ready to upload into CDX-EPA web site"); fileUploadStatus = true; } else if (excelFileRecordStr == "0") { Logger.WriteLog("No records to upload into CDX web site!"); System.Environment.Exit(0); } else if (excelFileRecordStr == "Error") { Logger.WriteLog("CDX - EPA file data extract process failed!"); System.Environment.Exit(0); } //================================================================================== //Send email upon CDX-EPA successful Upload files //================================================================================== SendMail SendEmail = new SendMail(); if (fileName != "CDXGetActivityFileData") { if (fileUploadStatus == true) { Logger.WriteLog("CDX - EPA " + fileName + " upload process successful!"); //========================================================================================= //Insert a record with CDXFileUpload process status into database ProcessStatusDetail table //========================================================================================= procesSts = "CDXFileUploadSuccess - Uploaded fileName: " + fileName; CDXFileLastUploadDate = GetFileData.getStoredProcData("CDXFileUploadStatus", procesSts, processBeginDate); Logger.WriteLog("Send email upon CDX-EPA successful Upload files "); Logger.WriteLog("================End Log Process================"); SendEmail.SendEmail("Add your email id", "CDX-EPA File Upload: " + fileName + " Success Auto Mail"); } else if (fileUploadStatus == false) { Logger.WriteLog("CDX - EPA Upload file process failed!"); Logger.WriteLog("================End Log Process================"); procesSts = "CDXFileUploadFailure"; //========================================================================================= //Insert a record with CDXFileUpload process status into database ProcessStatusDetail table //========================================================================================= CDXFileLastUploadDate = GetFileData.getStoredProcData("CDXFileUploadStatus", procesSts, processBeginDate); SendEmail.SendEmail("Add your email id", "CDX-EPA File Upload: " + fileName + " Failure Auto Mail"); } } System.Environment.Exit(0); Console.Read(); } catch (Exception e) { Exception e2 = new Exception("Upload a file to the web server", e); throw e2; } } } } ---==================================== --Program.cs - End ---==================================== ---==================================== --DataActions.cs - Start ---==================================== using System; using System.Collections.Generic; using System.Configuration; using System.Data; using System.Data.SqlClient; using System.IO; using System.Net; using System.Text; using ClosedXML.Excel; using System.Web.Script.Serialization; using System.Security.Cryptography; using Newtonsoft.Json.Linq; using System.Linq; using System.Threading.Tasks; namespace CDXFileUploadService { public class DataActions { //================================================================================== //Get Projects/Stations/Activity data set from SQL Server IMAP databases //=================================================================================== public string getStoredProcData(string storedProcName, string procStatus, DateTime procBeginDate) { DataSet ds = null; string procLastUpdatedDate = ""; string createActivityFileDataSts = ""; using (SqlConnection con = new SqlConnection(ConfigurationManager.ConnectionStrings["conn"].ConnectionString)) { try { SqlCommand cmd = new SqlCommand(storedProcName, con); cmd.CommandType = CommandType.StoredProcedure; if (storedProcName == "CDXFileUploadStatus") { cmd.Parameters.Add("@CDXFileLastUploadDate", SqlDbType.DateTime); cmd.Parameters["@CDXFileLastUploadDate"].Direction = ParameterDirection.Output; cmd.Parameters.AddWithValue("@CDXFileUploadStatus", procStatus); cmd.Parameters.AddWithValue("@ProcessBeginDate", procBeginDate); cmd.Parameters.AddWithValue("@ProcessEndDate", DateTime.Now); } else if (storedProcName == "CDXGetActivityFileData") { Logger.WriteLog("Executing Stored Procedure: " + storedProcName); cmd.Parameters.Add("@CrtActivityDataStatus", SqlDbType.NVarChar); cmd.Parameters["@CrtActivityDataStatus"].Size = 100; cmd.Parameters["@CrtActivityDataStatus"].Direction = ParameterDirection.Output; con.Open(); cmd.ExecuteNonQuery(); createActivityFileDataSts = Convert.ToString(cmd.Parameters["@CrtActivityDataStatus"].Value); con.Close(); Logger.WriteLog(" Inserting records into all Activity tables, creating Activity Analytical data for the Activity file was successful, returned success code: " + createActivityFileDataSts); //Net check for createActivityFileDataSts, if success then next extract data & create excel file if (createActivityFileDataSts == "LabParmsAndActivityDataAnalyticsSuccessful") { return createActivityFileDataSts; } } if (procStatus.Trim().Length>0) { con.Open(); cmd.ExecuteNonQuery(); procLastUpdatedDate = Convert.ToString(cmd.Parameters["@CDXFileLastUploadDate"].Value); con.Close(); Logger.WriteLog("Last CDXFileUpload date was: " + procLastUpdatedDate); Logger.WriteLog(" Inserting a record into CDXFileUploadStatus table with CDXFileUpload Status: " + procStatus); } else if (procStatus.Trim().Length == 0) { SqlDataAdapter da = new SqlDataAdapter(); da.SelectCommand = cmd; ds = new DataSet(); da.Fill(ds); } if (storedProcName == "CDXFileUploadProjectsData" || storedProcName == "CDXFileUploadStationsData" || storedProcName == "CDXFileUploadActivityData") { Logger.WriteLog("IMAP database connection successful"); Logger.WriteLog("Executing Stored Procedure: " + storedProcName); if (ds.Tables.Count > 0) { ExportDataSetToExcel(ds, storedProcName); return "1"; } else if (ds.Tables.Count == 0) { Logger.WriteLog(" There are no records to upload for CDX - EPA file upload process"); return "0"; } } return "1"; } catch (Exception e) { throw; Logger.WriteLog("SQL Server failed: " + e.Message); return "Error"; } finally { if (ds != null) { ds.Dispose(); } Logger.WriteLog("CDX - EPA process successfully executed stored procedure: "+ storedProcName); } } } //====================================== //Get Activity data //====================== public string getActivityData(string storedProcName, string procStatus, DateTime procBeginDate) { DataSet ds = null; //string procLastUpdatedDate = ""; string createActivityFileDataSts = ""; using (SqlConnection con = new SqlConnection(ConfigurationManager.ConnectionStrings["conn"].ConnectionString)) { try { SqlCommand cmd = new SqlCommand(storedProcName, con); cmd.CommandType = CommandType.StoredProcedure; if (storedProcName == "CDXGetActivityFileData") { Logger.WriteLog("Executing Stored Procedure: " + storedProcName); cmd.Parameters.Add("@CrtActivityDataStatus", SqlDbType.NVarChar); cmd.Parameters["@CrtActivityDataStatus"].Size = 100; cmd.Parameters["@CrtActivityDataStatus"].Direction = ParameterDirection.Output; con.Open(); cmd.ExecuteNonQuery(); createActivityFileDataSts = Convert.ToString(cmd.Parameters["@CrtActivityDataStatus"].Value); con.Close(); Logger.WriteLog(" Inserting records into all Activity tables, creating Activity Analytical data for the Activity file was successful, returned success code: " + createActivityFileDataSts); if (createActivityFileDataSts == "LabParmsAndActivityDataAnalyticsSuccessful") { return createActivityFileDataSts; } else { return "LabParmsAndActivityDataAnalyticsFailed"; } } return "2"; } catch (Exception e) { throw; Logger.WriteLog("SQL Server failed: " + e.Message); return "Error"; } finally { if (ds != null) { ds.Dispose(); } Logger.WriteLog("CDX - EPA process successfully executed stored procedure: " + storedProcName); } } } //==================== //End of Activity Data //==================== //================================================================================== //Create Projects, Stations and Activity Excel files for CDX-EPA data upload process //=================================================================================== private void ExportDataSetToExcel(DataSet ds, string ProcName) { string AppLocation = ""; AppLocation = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().CodeBase); AppLocation = AppLocation.Replace("file:\\", ""); string file = ""; string templateName = ""; if (ProcName == "CDXFileUploadProjectsData") { file = ConfigurationManager.AppSettings["filePath"]+"ProjectsDataFile.xlsx"; templateName = "ProjectData"; Logger.WriteLog("Begin Extracting IMAP data for creating a WQX_CDX_ProjectsData file, filepath is: " + file); } else if (ProcName == "CDXFileUploadStationsData") { file = ConfigurationManager.AppSettings["filePath"]+"StationsDataFile.xlsx"; templateName = "StationsData"; Logger.WriteLog("Extracting IMAP data for creating a StationsDataFile.xlsx file, filepath is: " + file); } else if (ProcName == "CDXFileUploadActivityData") { file = ConfigurationManager.AppSettings["filePath"] + "MasterActivityDataFile.xlsx"; //MasterActivityDataFile templateName = "ActivityDataFile"; Logger.WriteLog("Extracting IMAP data for creating a MasterActivityDataFile.xlsx file, filepath is: " + file); } using (XLWorkbook wb = new XLWorkbook()) { // var ws = wb.Worksheets.Add(templateName); wb.Worksheets.Add(ds.Tables[0]); wb.Style.Alignment.Horizontal = XLAlignmentHorizontalValues.Center; wb.Style.Font.Bold = true; wb.SaveAs(file); // logger.Trace("Excel data file created on filepath: " + AppLocation); } Logger.WriteLog("ExportDataSetToExcel process complete"); } //================================================================================== //CDX-EPA Projects,Stations and Activity files upload process //=================================================================================== public async Task CDXfileUploadProcessAsync(string dataFileName,string CDXfileName, string queryString) { bool response = false; // This user id and private encryption key are for illustration purposes only. string userID = "TestUser"; string privateKey = "TestKey"; //string fName = fileName; var arguments = new Dictionary(); using (WebClient client = new WebClient()) { try { // get prams //Test this str and check out string tstQry = string.Empty; string timeStamp = ""; string data = ""; string fileId = ""; string datasetId = ""; byte[] signature; Uri uri; //****************************UPLOAD A FILE************************ //*******Upload a file to the web server -- Returns fileId********* try { uri = new Uri("https://cdx.epa.gov/WQXWeb/api/Upload/"+ dataFileName); client.Headers.Add("X-UserID", userID); timeStamp = DateTime.UtcNow.ToString(); client.Headers.Add("X-Stamp", timeStamp); // The encrypted data must be in this order and format data = String.Format("{0}{1}{2}{3}", userID, timeStamp, uri.ToString(), "POST"); signature = Encoding.UTF8.GetBytes(data); using (HMACSHA256 hmac = new HMACSHA256(Convert.FromBase64String(privateKey))) { byte[] signatureBytes = hmac.ComputeHash(signature); client.Headers.Add("X-Signature", Convert.ToBase64String(signatureBytes)); string s = System.Text.Encoding.ASCII.GetString(signature); } // Decide what type of content to be returned (text/plain, json, xml) client.Headers.Add("Content-Type", "text/plain"); FileStream sourceFile = new FileStream(CDXfileName, FileMode.Open); BinaryReader binReader = new BinaryReader(sourceFile); byte[] fileInput = new byte[sourceFile.Length]; //create byte array of size file for (int i = 0; i < sourceFile.Length; i++) fileInput[i] = binReader.ReadByte(); //read until done sourceFile.Close(); //dispose of stream binReader.Close(); //dispose of reader fileInput = client.UploadData(uri, fileInput); fileId = System.Text.Encoding.UTF8.GetString(fileInput); fileId = fileId.Substring(1, fileId.Length - 2); Logger.WriteLog("CDX-EPA - Upload a file to the web server -- Returns fileId: " + fileId); } catch (WebException e) { Exception e2 = new Exception(((HttpWebResponse)e.Response).StatusDescription, e); throw e2; } catch (Exception e) { Exception e2 = new Exception("Upload a file to the web server", e); throw e2; // Logger.WriteLog("Error"+ e2); } try { string qString = queryString + fileId; //Uncomment and change to Test query string while testing this process // qString = qString + "&generatedElementName1=Organization ID&generatedElementValue1=WQXTEST"; //Production query string qString = qString + "&generatedElementName1=Organization ID&generatedElementValue1=21SC60WQ_WQX"; client.Headers.Clear(); Console.WriteLine("Appended qString :" + qString); //URI of StartImport. get datasetId uri = new Uri("https://cdx.epa.gov/WQXWeb/api/StartImport?" + qString); string inputJson = (new JavaScriptSerializer()).Serialize(qString); client.Headers.Add("X-UserID", userID); timeStamp = DateTime.UtcNow.ToString(); client.Headers.Add("X-Stamp", timeStamp); data = String.Format("{0}{1}{2}{3}", userID, timeStamp, uri.ToString(), "GET"); // data = String.Format("{0}{1}{2}{3}{4}", userID, timeStamp, uri.ToString(), "GET", client.QueryString); signature = Encoding.UTF8.GetBytes(data); //Test using (HMACSHA256 hmac = new HMACSHA256(Convert.FromBase64String(privateKey))) { byte[] signatureBytes = hmac.ComputeHash(signature); client.Headers.Add("X-Signature", Convert.ToBase64String(signatureBytes, Base64FormattingOptions.None)); datasetId = Encoding.ASCII.GetString(client.DownloadData(uri)); Console.WriteLine(datasetId); Logger.WriteLog("CDX-EPA - Upload a file to the web server -- Returns datasetId: " + datasetId); } } catch (WebException e) { Exception e2 = new Exception(((HttpWebResponse)e.Response).StatusDescription + ", " + e.Status.ToString(), e); throw e2; } catch (Exception e) { Exception e2 = new Exception("Start importing a file and get back a dataset ID Process Failed:", e); throw e2; } datasetId = datasetId.Substring(1, datasetId.Length - 2); // Decide what type of content to be returned (json, xml) client.Headers.Add("Content-Type", "application/json"); //****************************GET STATUS OF DATASET************************ client.Headers.Clear(); try { //The URI of the get dataset status webservice with parameters uri = new Uri("https://cdx.epa.gov/WQXWeb/api/GetStatus?datasetId=" + datasetId); // uri = new Uri("https://cdx.epa.gov/WQXWeb/api/SubmitDatasetToCdx?datasetId=" + datasetId); client.Headers.Add("X-UserID", userID); timeStamp = DateTime.UtcNow.ToString(); client.Headers.Add("X-Stamp", timeStamp); // The encrypted data must be in this order and format data = String.Format("{0}{1}{2}{3}", userID, timeStamp, uri.ToString(), "GET"); signature = Encoding.UTF8.GetBytes(data); using (HMACSHA256 hmac = new HMACSHA256(Convert.FromBase64String(privateKey))) { byte[] signatureBytes = hmac.ComputeHash(signature); client.Headers.Add("X-Signature", Convert.ToBase64String(signatureBytes, Base64FormattingOptions.None)); } // Decide what type of content to be returned (json, xml) //client.Headers.Add("Content-Type", "application/xml"); client.Headers.Add("Content-Type", "application/json"); // Process the response // Byte[] httpMessage = client.DownloadData(uri); // Byte[] httpMessage = await client.DownloadDataTaskAsync(uri); // if (httpMessage.Length > 0) { Console.WriteLine(Encoding.ASCII.GetString(httpMessage)); JObject originalObject = JObject.Parse(Encoding.ASCII.GetString(httpMessage)); var CDXFileUploadStatus = originalObject.Descendants() .OfType() .Where(p => p.Name == "StatusName") .Select(x => x.Value.ToString()) .ToArray(); Console.WriteLine(string.Join(", ", CDXFileUploadStatus)); Logger.WriteLog("======CDX-EPA-Upload File Status====== : " + "\n" + JObject.Parse(Encoding.ASCII.GetString(httpMessage))); if (string.Join(", ", CDXFileUploadStatus) == "Import Failed") { response = false; return response; } else { response = true; return response; } } response= true; return response; } catch (WebException e) { Exception e2 = new Exception(((HttpWebResponse)e.Response).StatusDescription + ", " + e.Status.ToString(), e); throw e2; } catch (Exception e) { Exception e2 = new Exception("GetStatus Failed", e); throw e2; } } catch (Exception e) { Console.WriteLine(e.Message); Console.Read(); return false; } } } } } ---==================================== --DataActions.cs - End ---==================================== ---==================================== --Logger.cs - Start ---==================================== using System; using System.Collections.Generic; using System.Text; using System.Configuration; using System.IO; using System.Linq; using System.Threading.Tasks; namespace CDXFileUploadService { public static class Logger { private static string lPath = ConfigurationManager.AppSettings["logPath"]; public static void WriteLog(string message) { using (StreamWriter writer = new StreamWriter(lPath, true)) { writer.WriteLine($"{DateTime.Now}:{message}"); } } } } ---==================================== --Logger.cs - End ---==================================== ---==================================== --SendMail.cs - Start ---==================================== using System; using System.Collections.Generic; using System.Configuration; using System.Linq; using System.Net; using System.Net.Mail; using System.Text; using System.Threading.Tasks; namespace CDXFileUploadService { public class SendMail { //Send Email public void SendEmail(string MailTo, string MailSubject) { try { //Get Excel path and attach this file to mail string AppLocation = ""; AppLocation = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().CodeBase); AppLocation = AppLocation.Replace("file:\\", ""); string logfile = ConfigurationManager.AppSettings["logPath"]; MailMessage mail = new MailMessage(); SmtpClient SmtpServer = new SmtpClient(ConfigurationManager.AppSettings["Host"].ToString()); mail.From = new MailAddress(ConfigurationManager.AppSettings["MailFrom"].ToString()); mail.To.Add(MailTo); // Sending MailTo List mailLst = new List(); mailLst.Add("Add your email id"); mail.CC.Add(string.Join(",", mailLst)); // Sending CC mail.Bcc.Add(string.Join(",", mailLst)); // Sending Bcc mail.Subject = MailSubject; // Mail Subject mail.Body = "*This is an automatically generated email, please do not reply* DHEC - [CDX-WQX] - Upload Process email notification, review logs for details!*"; System.Net.Mail.Attachment attachment; attachment = new System.Net.Mail.Attachment(logfile); //Attaching File to Mail mail.Attachments.Add(attachment); SmtpServer.Port = Convert.ToInt32(ConfigurationManager.AppSettings["Port"]); //PORT SmtpServer.EnableSsl = true; SmtpServer.DeliveryMethod = SmtpDeliveryMethod.Network; SmtpServer.UseDefaultCredentials = false; SmtpServer.Credentials = new NetworkCredential("Email id of add your email id", "Password of add your email id"); SmtpServer.Send(mail); } catch (Exception ex) { throw ex; } } //Send Email } } ---==================================== --SendMail.cs - End ---==================================== ---==================================== --App.config - Start ---==================================== ---==================================== --App.config - End ---====================================