如何从csv文件中只导入不存在的行并将现有行更新到SQL数据库C#

p4rjhz4m  于 2023-02-06  发布在  C#
关注(0)|答案(1)|浏览(162)

在sylvan.data.csv包的帮助下,我现在可以将sql表中的scheme应用到csv中了。但是,当我想检查csv中的一行是否存在于sql数据库中时,会出现以下问题。如果存在,则需要更新该行,如果不存在,则需要导入不存在的行。但使用批量复制时,这是不可能的。
我得到了下面的代码:

static void LoadTableCsv(SqlConnection conn, string tableName, string csvFile)
        {

            // read the column schema of the target table
            var cmd = conn.CreateCommand();
            conn.Open();
            cmd.CommandText = $"select top 0 * from {tableName}"; // beware of sql injection
            var reader = cmd.ExecuteReader();
            var colSchema = reader.GetColumnSchema();
            reader.Close();

            // apply the column schema to the csv reader.
            var csvSchema = new CsvSchema(colSchema);
            var csvOpts = new CsvDataReaderOptions { Schema = csvSchema };
            using var csv = CsvDataReader.Create(csvFile, csvOpts);

            // Initialize SqlCommand for checking if the record already exists.
            using var checkCommand = new SqlCommand("SELECT COUNT(*) FROM {tablename} WHERE TicketID = @value", conn);
            checkCommand.Parameters.Add("@value", SqlDbType.Int, 10, "TicketID");

            using var bulkCopy = new SqlBulkCopy(conn);
            bulkCopy.DestinationTableName = tableName;
            bulkCopy.EnableStreaming = true;

            // Iterate through the records in the CSV file.
            while (csv.Read())
            {
                // Set the value of the "@value" parameter
                checkCommand.Parameters["@value"].Value = csv["TicketID"].ToString();

                // Execute the check command to see if the record already exists.
                var checkResult = (int)checkCommand.ExecuteScalar();
                if (checkResult == 0)
                {
                    // The record does not exist, write it to the SQL database using SqlBulkCopy.
                    bulkCopy.WriteToServer(new[] { csv });
                }
                else
                {
                    // The record already exists, update it using an UPDATE statement.
                    using var updateCommand = new SqlCommand("UPDATE {tablename} SET Column1 = @col1, Column2 = @col2, Column3 = @col3, Column4 = @col4, Column5 = @col5, Column6 = @col6 WHERE TicketID = @value", conn);

                    // Add parameters for each column you want to update, using the names and types of the columns in the target table.
                    updateCommand.Parameters.Add("@col1", SqlDbType.Int, 10, "TicketID");
                    updateCommand.Parameters.Add("@col2", SqlDbType.NVarChar, 50, "TicketTitle");
                    updateCommand.Parameters.Add("@col3", SqlDbType.NVarChar, 50, "TicketStatus");
                    updateCommand.Parameters.Add("@col4", SqlDbType.NVarChar, 50, "CustomerName");
                    updateCommand.Parameters.Add("@col5", SqlDbType.NVarChar, 50, "TechnicianFullName");
                    updateCommand.Parameters.Add("@col6", SqlDbType.DateTime, 50, "TicketResolvedDate");
                    updateCommand.Parameters.Add("@value", SqlDbType.Int, 10, "TicketID");

                    // Set the values of the parameters to the values in the current row of the CSV file.
                    updateCommand.Parameters["@col1"].Value = int.Parse(csv["TicketID"].ToString());
                    updateCommand.Parameters["@col2"].Value = csv["TicketTitle"].ToString();
                    updateCommand.Parameters["@col3"].Value = csv["TicketStatus"].ToString();
                    updateCommand.Parameters["@col4"].Value = csv["CustomerName"].ToString();
                    updateCommand.Parameters["@col5"].Value = csv["TechnicianFullName"].ToString();
                    updateCommand.Parameters["@col6"].Value = DateTime.Parse(csv["TicketResolvedDate"].ToString());
                    updateCommand.Parameters["@value"].Value = int.Parse(csv["TicketID"].ToString());

                    // Execute the update command.
                    updateCommand.ExecuteNonQuery();
                }

            }
            conn.Close();
        }

但这给了我一个错误,因为批量复制不能只读取一个数据行。

goqiplq2

goqiplq21#

Siggermanen的评论是正确的建议,您应该将所有数据批量装载到临时表中,然后使用SQL命令(s)将临时表中的数据合并到目标表中。理想情况下,您可以使用T-Sql Merge Statement来执行此操作。您还可以使用单独的update和insert语句。这需要了解表列的知识,以便创建合并数据的命令。您 * 可以 *通过阅读查询表的INFORMATION_SCHEMA来确定列和键,并使用它来动态构造merge语句,动态地执行此操作。或者,如果您在编译时知道架构,则可以硬编码语句,这将大大简化开发和测试。

using Sylvan.Data.Csv;
using System.Data.SqlClient;

static void LoadTableCsv(SqlConnection conn, string tableName, string csvFile)
{
    // read the column schema of the target table
    var cmd = conn.CreateCommand();
    cmd.CommandText = $"select top 0 * from {tableName}"; // beware of sql injection
    var reader = cmd.ExecuteReader();
    var colSchema = reader.GetColumnSchema();
    reader.Close();

    // create a temp table to load the data into
    // using the destination table as a template
    cmd.CommandText = $"select top 0 * into #load from {tableName}";
    cmd.ExecuteNonQuery();

    // apply the column schema to the csv reader.
    var csvSchema = new CsvSchema(colSchema);
    var csvOpts = new CsvDataReaderOptions { Schema = csvSchema };
    using var csv = CsvDataReader.Create(csvFile, csvOpts);

    // push *all* data into the temp table
    using var bulkCopy = new SqlBulkCopy(conn);
    bulkCopy.DestinationTableName = "#load";
    bulkCopy.EnableStreaming = true;
    bulkCopy.WriteToServer(csv);

    // use sql commands to "MERGE" the data into the destination table
    cmd.CommandText = $"""
                        insert into {tableName}
                        select * from #load l
                        where not exists (select * from {tableName} d where d.Id = l.Id)
                       """;

    cmd.ExecuteNonQuery();

}

在循环中执行insert/update语句是您试图避免的。这会产生与数据库的“聊天”通信,性能将由每个操作的开销决定。相反,您希望尽可能地使操作“大块”,SqlBulkCopy和MERGE将提供这一点。

相关问题