功能齐全、效率一流的免费开源数据库导入导出工具（c#开发，支持SQL server、SQLite、ACCESS三种数据库），每月借此处理数据5G以上

浏览数：33 / 时间：2015年06月12日

软件名：DataPie

功能：支持SQL server、SQLite、ACCESS数据库的导入、导出、存储过程调用，支持EXCEL2007、EXCEL2003、ACCESS2007、 CSV文件导入数据库，支持EXCEL、CSV、ZIP、ACCESS文件方式导出，支持数据拆分导出及自定义SQL查询与导出。

开发背景：作者从事财务管理工作，主要是出具集团的内部财务报表，随着公司精细化管理的需求，管理报表的数据量急速增长，依赖EXCEL加工处理数据已经变得极为困难，因此团队全面转向关系数据库进行数据处理，为减少财务人员使用数据库的难度，因此专门针对财务报表核算需要，开发了该工具。目前，我月度报表处理的数据量超过5G，最大的单次运算量记录接近千万，该工具主要发挥的作用就是将收集到的数据，导入SQL SERVER数据库，进行报表运算，并且输出各类财务报表，对于几十万级的数据输入、输出基本上能够轻松应付。

源码下载地址：https://github.com/yfl8910/DataPie

软件界面及主要代码：

1.导入界面，支持EXCEL2007、EXCEL2003、ACCESS2007、 CSV文件导入数据库，支持CSV文件夹整体导入相应表。财务工作的数据源较多，有业务提供的excel数据，也有从系统中取出的csv、excel或者text文档，为满足多种数据源的输入，所以多种数据源导入数据库。目前SQL SERVER导入整体效率较高，ACCESS下，excel文件导入效率最高（几十万的数据量可以很快导入），其他方式效率还有待提升。

涉及核心代码如下

  public static DataTable GetDataTableFromFile(string path, string tname)
        {
            string ace = "Microsoft.ACE.OLEDB.12.0";
            string jet = "Microsoft.Jet.OLEDB.4.0";
            string xl2007 = "Excel 12.0 Xml";
            string xl2003 = "Excel 8.0";
            string imex = "IMEX=1";
            string hdr = "Yes";
            string conn = "Provider={0};Data Source={1};Extended Properties=\"{2};HDR={3};{4}\";";
            string select = "";
            string ext = Path.GetExtension(path);
            OleDbDataAdapter oda;
            DataTable dt = new DataTable(tname);
            switch (ext.ToLower())
            {
                case ".xlsx":
                    conn = String.Format(conn, ace, Path.GetFullPath(path), xl2007, hdr, imex);
                    select = string.Format("SELECT * FROM [{0}$]", tname);
                    break;
                case ".xls":
                    conn = String.Format(conn, jet, Path.GetFullPath(path), xl2003, hdr, imex);
                    select = string.Format("SELECT * FROM [{0}$]", tname);
                    break;
                case ".accdb":
                    conn = String.Format("Provider=Microsoft.ACE.OLEDB.12.0;Data Source= {0};Persist Security Info=False;", path);
                    select = string.Format("SELECT * FROM [{0}]", tname);
                    break;
                case ".csv":
                    conn = String.Format(conn, ace, path.Substring(0, path.LastIndexOf(‘\\‘)), "text;Excel 12.0", hdr, imex);
                    select = string.Format("SELECT * FROM [{0}]", Path.GetFileName(path));
                    break;
                default:
                    throw new Exception("File Not Supported!");
            }
            OleDbConnection con = new OleDbConnection(conn);
            con.Open();
            oda = new OleDbDataAdapter(select, con);
            oda.Fill(dt);
            con.Close();
            return dt;
        }

2.数据导出界面，支持EXCEL2007、CSV、ACCESS、zip压缩文件、分拆导出几个功能。我目前从事的岗位，报表整体数据量接近千万级，其中单表需要导出分发到业务手中的数据量最大也超过了百万级，目前达到百万级数据量的表导出，一般采用CSV文件或者压缩包的形式，或者分多个表导出。

涉及核心代码。excel导出主要源码：

  public static void SaveExcel(string FileName, string sql, string SheetName)
        {

            FileInfo newFile = new FileInfo(FileName);
            if (newFile.Exists)
            {
                newFile.Delete();
                newFile = new FileInfo(FileName);
            }
            using (ExcelPackage package = new ExcelPackage(newFile))
            {
                try
                {
                    ExcelWorksheet ws = package.Workbook.Worksheets.Add(SheetName);

                    IDataReader reader = DBConfig.db.DBProvider.ExecuteReader(sql);
                    ws.Cells["A1"].LoadFromDataReader(reader, true);
                }
                catch (Exception ex)
                {
                    throw ex;
                }
                package.Save();
            }
         
        }

        /// <summary>
        /// 工作簿中添加新的SHEET
        /// </summary>
        public static bool SaveExcel(ExcelPackage package, string sql, string SheetName)
        {

            try
            {
                ExcelWorksheet ws = package.Workbook.Worksheets.Add(SheetName);
                IDataReader reader = DBConfig.db.DBProvider.ExecuteReader(sql);
                ws.Cells["A1"].LoadFromDataReader(reader, true);

                return true;
            }
            catch (Exception ex)
            {
                throw ex;
            }
        }
        /// <summary>
        /// 单表格导出到一个EXCEL工作簿
        /// </summary>
        public static int ExportExcel(string FileName, string sql, string SheetName)
        {
            if (FileName != null)
            {
                Stopwatch watch = Stopwatch.StartNew();
                watch.Start();

                SaveExcel(FileName, sql, SheetName);
                watch.Stop();
                return Convert.ToInt32(watch.ElapsedMilliseconds / 1000);
            }

            return -1;
        }

        public static async Task<int> ExportExcelAsync(string FileName, string sql, string SheetName)
        {

            return await Task.Run(
                () => { return ExportExcel(FileName,sql,SheetName); }
                );

        }

        /// <summary>
        /// 多表格导出到一个EXCEL工作簿
        /// </summary>
        public static int ExportExcel(string[] TabelNameArray, string filename)
        {
            if (filename != null)
            {
                Stopwatch watch = Stopwatch.StartNew();
                watch.Start();

                FileInfo newFile = new FileInfo(filename);
                if (newFile.Exists)
                {
                    newFile.Delete();
                    newFile = new FileInfo(filename);
                }


                using (ExcelPackage package = new ExcelPackage(newFile))
                {
                    for (int i = 0; i < TabelNameArray.Length; i++)
                    {
                        string sql = "select * from  [" + TabelNameArray[i] + "]";
                        IDataReader reader = DBConfig.db.DBProvider.ExecuteReader(sql);
                        SaveExcel(package, sql, TabelNameArray[i]);

                    }
                    package.Save();
                }
                watch.Stop();
                return Convert.ToInt32(watch.ElapsedMilliseconds / 1000);
            }

            return -1;
        }

        public static  async Task<int> ExportExcelAsync(string[] TabelNameArray, string filename) {

            return await Task.Run(
                () => { return ExportExcel( TabelNameArray,filename); }      
                );

        }

        /// <summary>
        /// 分拆导出
        /// </summary>
        public static int ExportExcel(string[] TabelNameArray, string filename, string[] whereSQLArr)
        {
            if (filename != null)
            {
                Stopwatch watch = Stopwatch.StartNew();
                watch.Start();
                FileInfo file = new FileInfo(filename);
                int wherecount = whereSQLArr.Length;
                int count = TabelNameArray.Length;
                for (int i = 0; i < wherecount; i++)
                {
                    string s = filename.Substring(0, filename.LastIndexOf("\\"));
                    StringBuilder newfileName = new StringBuilder(s);
                    int index = whereSQLArr[i].LastIndexOf("=");
                    string sp = whereSQLArr[i].Substring(index + 2, whereSQLArr[i].Length - index - 3);
                    newfileName.Append("\\" + file.Name.Substring(0, file.Name.LastIndexOf(".")) + "_" + sp + ".xlsx");
                    FileInfo newFile = new FileInfo(newfileName.ToString());
                    if (newFile.Exists)
                    {
                        newFile.Delete();
                        newFile = new FileInfo(newfileName.ToString());
                    }

                    for (int j = 0; j < count; j++)
                    {
                        string sql = "select * from [" + TabelNameArray[j] + "]" + whereSQLArr[i];
                        IDataReader reader = DBConfig.db.DBProvider.ExecuteReader(sql);
                        SaveExcel(newfileName.ToString(), sql, TabelNameArray[j]);

                    }


                }

                watch.Stop();
                return Convert.ToInt32(watch.ElapsedMilliseconds / 1000);
            }

            return -1;
        }

        public static async Task<int> ExportExcelAsync(string[] TabelNameArray, string filename, string[] whereSQLArr)
        {

            return await Task.Run(
                () => { return ExportExcel(TabelNameArray,filename,whereSQLArr); }
                );

        }

csv导出主要源码（其他代码请参github上的源码）：

  private static void WriteHeader(IDataReader reader, StreamWriter sw)
        {
            for (int i = 0; i < reader.FieldCount; i++)
            {
                if (i > 0)
                    sw.Write(‘,‘);
                sw.Write(reader.GetName(i));
            }
            sw.Write(Environment.NewLine);
        }
        private static void WriteContent(IDataReader reader, StreamWriter sw)
        {
            for (int i = 0; i < reader.FieldCount; i++)
            {
                if (i > 0)
                    sw.Write(‘,‘);
                String v = reader[i].ToString();
                if (v.Contains(‘,‘) || v.Contains(‘\n‘) || v.Contains(‘\r‘) || v.Contains(‘"‘))
                {
                    sw.Write(‘"‘);
                    sw.Write(v.Replace("\"", "\"\""));
                    sw.Write(‘"‘);
                }
                else
                {
                    sw.Write(v);
                }
            }
            sw.Write(Environment.NewLine);
        }

        public static int SaveCsv(IDataReader reader, string filename)
        {
            Stopwatch watch = Stopwatch.StartNew();
            watch.Start();

            StreamWriter sw = new StreamWriter(filename, false, Encoding.GetEncoding("gb2312"));
            WriteHeader(reader, sw);

            while (reader.Read())
            {
                WriteContent(reader, sw);
            }

            sw.Flush();
            sw.Close();//释放资源

            watch.Stop();
            return Convert.ToInt32(watch.ElapsedMilliseconds / 1000);
        }
        public static async Task<int> ExportCsvAsync(IDataReader reader, string filename)
        {
            return await Task.Run( () => { return SaveCsv( reader,  filename);} );
        }

        public static StreamWriter GetStreamWriter(string filename, int outCount)
        {
            string s = filename.Substring(0, filename.LastIndexOf("."));
            StringBuilder newfileName = new StringBuilder(s);
            newfileName.Append(outCount + 1 + ".csv");
            FileInfo newFile = new FileInfo(newfileName.ToString());
            if (newFile.Exists)
            {
                newFile.Delete();
                newFile = new FileInfo(newfileName.ToString());
            }
            StreamWriter sw = new StreamWriter(newfileName.ToString(), false, Encoding.GetEncoding("gb2312"));
            return sw;
        }
    
        public static int SaveCsv(IDataReader reader, string filename, int pagesize)
        {
            int innerCount = 0, outCount = 0;
            Stopwatch watch = Stopwatch.StartNew();
            watch.Start();
            StreamWriter sw = GetStreamWriter(filename, outCount);
            WriteHeader(reader, sw);
            while (reader.Read())
            {
                if (innerCount < pagesize)
                {
                    WriteContent(reader, sw);
                    innerCount++;
                }
                else
                {
                    innerCount = 0;
                    outCount++;
                    sw.Flush();
                    sw.Close();
                    sw = GetStreamWriter(filename, outCount);
                    WriteHeader(reader, sw);
                    WriteContent(reader, sw);
                    innerCount++;
                }
            }
            sw.Flush();
            sw.Close();
            watch.Stop();
            return Convert.ToInt32(watch.ElapsedMilliseconds / 1000);

        }
        public static async Task<int> ExportCsvAsync(IDataReader reader, string filename, int pagesize)
        {
            return await Task.Run(() => { return SaveCsv(reader, filename,pagesize); });
        }